hermann 0.26.0.0 → 0.26.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1ff1d84d0e2dcf323ab815f629e08d5459d7a462
-  data.tar.gz: df848b706e5e7ae27453a5ea184b0c02d9a07469
+  metadata.gz: 7a8c3c2350a07ffdafa0a2aaba48525f6ed12322
+  data.tar.gz: 9b6211fb45622bbbe676511014f76b3da1b74c8c
 SHA512:
-  metadata.gz: 5049f62844ddaf48024efe4f7084a3b5cf0efb192ddf14a47ef1c9d5a3771d083ecdbcbf1f456f6d761fc7ebe9826003ab3d4ec4613238fe66b17b98c61ec8ec
-  data.tar.gz: f5b2e3591b4c5a3458a664897643a67c08880ff2f528b40bd94ef68a85a10644cff7fd4a7c37cdd295c338369d0fd5a7e57c4a57091a264285e13fc31a8bce24
+  metadata.gz: 43fb663d9a068d3e425c08fcdb31cb17feaaa70a89b1ee46a9a48915eb76d7a6f025d2a0351c166b1f190bbe3fe7c6956bd61eeae0feb3928abd564985c717a5
+  data.tar.gz: a349bedefd031ce601e224755dc4d2e528c45e5bd43d3fc5b2ce9b74aae399fed3a77321402f6beca3bc8cf741d3de0a2186491e7dc4df683d9cef1ac52f207a

data/ext/hermann/hermann_rdkafka.c

@@ -33,6 +33,11 @@
 
 #include "hermann_rdkafka.h"
 
+/* This header file exposes the functions in librdkafka.a that are needed for
+ * consistent partitioning. After librdkafka releases a new tag and Hermann
+ * points to it, this can be removed. */
+#include "rdcrc32.h"
+
 #ifdef HAVE_RUBY_VERSION_H
 #include <ruby/version.h>
 #endif
@@ -134,6 +139,18 @@ static void msg_delivered(rd_kafka_t *rk,
 	}
 }
 
+
+/* This function is in rdkafka.h on librdkafka master. As soon as a new
+ * version is released and Hermann points to it, this can be removed. */
+int32_t rd_kafka_msg_partitioner_consistent (const rd_kafka_topic_t *rkt,
+											 const void *key, size_t keylen,
+											 int32_t partition_cnt,
+											 void *rkt_opaque,
+											 void *msg_opaque) {
+	return rd_crc32(key, keylen) % partition_cnt;
+}
+
+
 /**
  * Producer partitioner callback.
  * Used to determine the target partition within a topic for production.
@@ -154,17 +171,11 @@ static int32_t producer_partitioner_callback(const rd_kafka_topic_t *rkt,
 											 int32_t partition_cnt,
 											 void *rkt_opaque,
 											 void *msg_opaque) {
-	/* Pick a random partition */
-	int retry = 0;
-	int32_t partition = RD_KAFKA_PARTITION_UA;
-
-	for (; retry < partition_cnt; retry++) {
-		partition = rand() % partition_cnt;
-		if (rd_kafka_topic_partition_available(rkt, partition)) {
-			break; /* this one will do */
-		}
+	if (keylen) {
+		return rd_kafka_msg_partitioner_consistent(rkt, keydata, keylen, partition_cnt, rkt_opaque, msg_opaque);
+	} else {
+		return rd_kafka_msg_partitioner_random(rkt, keydata, keylen, partition_cnt, rkt_opaque, msg_opaque);
 	}
-	return partition;
 }
 
 /**
@@ -589,10 +600,11 @@ void producer_init_kafka(VALUE self, HermannInstanceConfig* config) {
  *  @param  message VALUE   the ruby String containing the outgoing message.
  *  @param  topic   VALUE   the ruby String containing the topic to use for the
  *							outgoing message.
+ *  @param  key  VALUE   the ruby String containing the key to partition by
  *  @param  result  VALUE   the Hermann::Result object to be fulfilled when the
  *		push completes
  */
-static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE result) {
+static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE partition_key, VALUE result) {
 
 	HermannInstanceConfig* producerConfig;
 	/* Context pointer, pointing to `result`, for the librdkafka delivery
@@ -600,6 +612,7 @@ static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE
 	 */
 	hermann_push_ctx_t *delivery_ctx = (hermann_push_ctx_t *)malloc(sizeof(hermann_push_ctx_t));
 	rd_kafka_topic_t *rkt = NULL;
+	rd_kafka_topic_conf_t *rkt_conf = NULL;
 
 	TRACER("self: %p, message: %p, result: %p)\n", self, message, result);
 
@@ -622,9 +635,15 @@ static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE
 
 	TRACER("kafka initialized\n");
 
+	/* Topic configuration */
+	rkt_conf = rd_kafka_topic_conf_new();
+
+	/* Set the partitioner callback */
+	rd_kafka_topic_conf_set_partitioner_cb(rkt_conf, producer_partitioner_callback);
+
 	rkt = rd_kafka_topic_new(producerConfig->rk,
 								RSTRING_PTR(topic),
-								NULL);
+								rkt_conf);
 
 	if (NULL == rkt) {
 		rb_raise(rb_eRuntimeError, "Could not construct a topic structure");
@@ -645,8 +664,8 @@ static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE
 						 RD_KAFKA_MSG_F_COPY,
 						 RSTRING_PTR(message),
 						 RSTRING_LEN(message),
-						 NULL,
-						 0,
+						 RSTRING_PTR(partition_key),
+						 RSTRING_LEN(partition_key),
 						 delivery_ctx)) {
 		fprintf(stderr, "%% Failed to produce to topic %s partition %i: %s\n",
 					rd_kafka_topic_name(producerConfig->rkt), producerConfig->partition,
@@ -1240,7 +1259,7 @@ void Init_hermann_rdkafka() {
 	rb_define_method(c_producer, "initialize_copy", producer_init_copy, 1);
 
 	/* Producer.push_single(msg) */
-	rb_define_method(c_producer, "push_single", producer_push_single, 3);
+	rb_define_method(c_producer, "push_single", producer_push_single, 4);
 
 	/* Producer.tick */
 	rb_define_method(c_producer, "tick", producer_tick, 1);

data/ext/hermann/rdcrc32.h

@@ -0,0 +1,103 @@
+/**
+ * \file rdcrc32.h
+ * Functions and types for CRC checks.
+ *
+ * Generated on Tue May  8 17:36:59 2012,
+ * by pycrc v0.7.10, http://www.tty1.net/pycrc/
+ *
+ * NOTE: Contains librd modifications:
+ *       - rd_crc32() helper.
+ *       - __RDCRC32___H__ define (was missing the '32' part).
+ *
+ * using the configuration:
+ *    Width        = 32
+ *    Poly         = 0x04c11db7
+ *    XorIn        = 0xffffffff
+ *    ReflectIn    = True
+ *    XorOut       = 0xffffffff
+ *    ReflectOut   = True
+ *    Algorithm    = table-driven
+ *****************************************************************************/
+#ifndef __RDCRC32___H__
+#define __RDCRC32___H__
+
+#include <stdlib.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/**
+ * The definition of the used algorithm.
+ *****************************************************************************/
+#define CRC_ALGO_TABLE_DRIVEN 1
+
+
+/**
+ * The type of the CRC values.
+ *
+ * This type must be big enough to contain at least 32 bits.
+ *****************************************************************************/
+typedef uint32_t rd_crc32_t;
+
+
+/**
+ * Reflect all bits of a \a data word of \a data_len bytes.
+ *
+ * \param data         The data word to be reflected.
+ * \param data_len     The width of \a data expressed in number of bits.
+ * \return             The reflected data.
+ *****************************************************************************/
+rd_crc32_t rd_crc32_reflect(rd_crc32_t data, size_t data_len);
+
+
+/**
+ * Calculate the initial crc value.
+ *
+ * \return     The initial crc value.
+ *****************************************************************************/
+static inline rd_crc32_t rd_crc32_init(void)
+{
+    return 0xffffffff;
+}
+
+
+/**
+ * Update the crc value with new data.
+ *
+ * \param crc      The current crc value.
+ * \param data     Pointer to a buffer of \a data_len bytes.
+ * \param data_len Number of bytes in the \a data buffer.
+ * \return         The updated crc value.
+ *****************************************************************************/
+rd_crc32_t rd_crc32_update(rd_crc32_t crc, const unsigned char *data, size_t data_len);
+
+
+/**
+ * Calculate the final crc value.
+ *
+ * \param crc  The current crc value.
+ * \return     The final crc value.
+ *****************************************************************************/
+static inline rd_crc32_t rd_crc32_finalize(rd_crc32_t crc)
+{
+    return crc ^ 0xffffffff;
+}
+
+
+/**
+ * Wrapper for performing CRC32 on the provided buffer.
+ */
+static inline rd_crc32_t rd_crc32 (const char *data, size_t data_len) {
+  return rd_crc32_finalize(rd_crc32_update(rd_crc32_init(),
+             (const unsigned char *)data,
+             data_len));
+}
+
+#ifdef __cplusplus
+}           /* closing brace for extern "C" */
+#endif
+
+#endif      /* __RDCRC32___H__ */

data/lib/hermann/producer.rb

@@ -63,8 +63,7 @@ module Hermann
       end
 
       if Hermann.jruby?
-        key = opts.has_key?(:partition_key) ? opts[:partition_key].to_java : nil
-        result = @internal.push_single(value, topic, key)
+        result = @internal.push_single(value, topic, opts[:partition_key], nil)
         unless result.nil?
           @children << result
         end
@@ -76,7 +75,7 @@ module Hermann
         # librdkafka callback queue overflow
         tick_reactor
         result = create_result
-        @internal.push_single(value, topic, result)
+        @internal.push_single(value, topic, opts[:partition_key].to_s, result)
       end
 
       return result

data/lib/hermann/provider/java_producer.rb

@@ -43,7 +43,8 @@ module Hermann
       # @return +Concurrent::Promise+ Representa a promise to send the
       #   data to the kafka broker.  Upon execution the Promise's status
       #   will be set
-      def push_single(msg, topic, key)
+      def push_single(msg, topic, key, _)
+        key = key && key.to_java
         Concurrent::Promise.execute {
           data = ProducerUtil::KeyedMessage.new(topic, nil, key, msg.to_java_bytes)
           begin

data/lib/hermann/version.rb

@@ -1,3 +1,3 @@
 module Hermann
-  VERSION = '0.26.0'
+  VERSION = '0.26.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hermann
 version: !ruby/object:Gem::Version
-  version: 0.26.0.0
+  version: 0.26.1.0
 platform: ruby
 authors:
 - R. Tyler Croy
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-08 00:00:00.000000000 Z
+date: 2015-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -68,6 +68,7 @@ files:
 - ext/hermann/extconf.rb
 - ext/hermann/hermann_rdkafka.c
 - ext/hermann/hermann_rdkafka.h
+- ext/hermann/rdcrc32.h
 - lib/hermann.rb
 - lib/hermann/consumer.rb
 - lib/hermann/discovery/metadata.rb
@@ -102,9 +103,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 3
 summary: A Kafka consumer/producer gem supporting both MRI and JRuby
 test_files: []
-has_rdoc: