hermann 0.26.0.0-java → 0.26.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49280cf6226fa7b24a9addc910823bae17cb06f4
-  data.tar.gz: d736fcac5e71f3e02d6753504d4f1b3b890be4d6
+  metadata.gz: 3695bb3cfd9f34a24690d0944f8c0a791c0a592c
+  data.tar.gz: b4915f4a6e67eea046c7413fb4899cb968b34d8a
 SHA512:
-  metadata.gz: c4f3460b2be2137b2e4b39553ce552316c5abe7c91fc018e336d1d66680461233b39cb4dcf2766fe1848479d81208a649004f0be34235d61c34318f3edfea128
-  data.tar.gz: 6f72c633e304be050f3abacf75622d00c01fd01ce936d0077a72df87f9493f48b146d6be4e3049611d7498aa6dfe3bccdbb3aca827adfa4b5cd9d8dbe04dba1b
+  metadata.gz: f17e8e04a4ee0952bad7c38d627544b567af01caf342b153074be595f1a0bacb45c86de0356e10dc3b2cffcc8c71e9507b3f1fa3c14789c35ef5f2de94f13eb2
+  data.tar.gz: ac5177acfa44c01028b3731ecf4a36b8a48dc97e7a126feed5941c4505d0e3a14f24c39f87127d272a0b41bd330325d3c21b16d88bfcaec8231ed3ed69e1eccc

data/ext/hermann/hermann_rdkafka.c

@@ -33,6 +33,11 @@
 
 #include "hermann_rdkafka.h"
 
+/* This header file exposes the functions in librdkafka.a that are needed for
+ * consistent partitioning. After librdkafka releases a new tag and Hermann
+ * points to it, this can be removed. */
+#include "rdcrc32.h"
+
 #ifdef HAVE_RUBY_VERSION_H
 #include <ruby/version.h>
 #endif
@@ -134,6 +139,18 @@ static void msg_delivered(rd_kafka_t *rk,
 	}
 }
 
+
+/* This function is in rdkafka.h on librdkafka master. As soon as a new
+ * version is released and Hermann points to it, this can be removed. */
+int32_t rd_kafka_msg_partitioner_consistent (const rd_kafka_topic_t *rkt,
+											 const void *key, size_t keylen,
+											 int32_t partition_cnt,
+											 void *rkt_opaque,
+											 void *msg_opaque) {
+	return rd_crc32(key, keylen) % partition_cnt;
+}
+
+
 /**
  * Producer partitioner callback.
  * Used to determine the target partition within a topic for production.
@@ -154,17 +171,11 @@ static int32_t producer_partitioner_callback(const rd_kafka_topic_t *rkt,
 											 int32_t partition_cnt,
 											 void *rkt_opaque,
 											 void *msg_opaque) {
-	/* Pick a random partition */
-	int retry = 0;
-	int32_t partition = RD_KAFKA_PARTITION_UA;
-
-	for (; retry < partition_cnt; retry++) {
-		partition = rand() % partition_cnt;
-		if (rd_kafka_topic_partition_available(rkt, partition)) {
-			break; /* this one will do */
-		}
+	if (keylen) {
+		return rd_kafka_msg_partitioner_consistent(rkt, keydata, keylen, partition_cnt, rkt_opaque, msg_opaque);
+	} else {
+		return rd_kafka_msg_partitioner_random(rkt, keydata, keylen, partition_cnt, rkt_opaque, msg_opaque);
 	}
-	return partition;
 }
 
 /**
@@ -589,10 +600,11 @@ void producer_init_kafka(VALUE self, HermannInstanceConfig* config) {
  *  @param  message VALUE   the ruby String containing the outgoing message.
  *  @param  topic   VALUE   the ruby String containing the topic to use for the
  *							outgoing message.
+ *  @param  key  VALUE   the ruby String containing the key to partition by
  *  @param  result  VALUE   the Hermann::Result object to be fulfilled when the
  *		push completes
  */
-static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE result) {
+static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE partition_key, VALUE result) {
 
 	HermannInstanceConfig* producerConfig;
 	/* Context pointer, pointing to `result`, for the librdkafka delivery
@@ -600,6 +612,7 @@ static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE
 	 */
 	hermann_push_ctx_t *delivery_ctx = (hermann_push_ctx_t *)malloc(sizeof(hermann_push_ctx_t));
 	rd_kafka_topic_t *rkt = NULL;
+	rd_kafka_topic_conf_t *rkt_conf = NULL;
 
 	TRACER("self: %p, message: %p, result: %p)\n", self, message, result);
 
@@ -622,9 +635,15 @@ static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE
 
 	TRACER("kafka initialized\n");
 
+	/* Topic configuration */
+	rkt_conf = rd_kafka_topic_conf_new();
+
+	/* Set the partitioner callback */
+	rd_kafka_topic_conf_set_partitioner_cb(rkt_conf, producer_partitioner_callback);
+
 	rkt = rd_kafka_topic_new(producerConfig->rk,
 								RSTRING_PTR(topic),
-								NULL);
+								rkt_conf);
 
 	if (NULL == rkt) {
 		rb_raise(rb_eRuntimeError, "Could not construct a topic structure");
@@ -645,8 +664,8 @@ static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE
 						 RD_KAFKA_MSG_F_COPY,
 						 RSTRING_PTR(message),
 						 RSTRING_LEN(message),
-						 NULL,
-						 0,
+						 RSTRING_PTR(partition_key),
+						 RSTRING_LEN(partition_key),
 						 delivery_ctx)) {
 		fprintf(stderr, "%% Failed to produce to topic %s partition %i: %s\n",
 					rd_kafka_topic_name(producerConfig->rkt), producerConfig->partition,
@@ -1240,7 +1259,7 @@ void Init_hermann_rdkafka() {
 	rb_define_method(c_producer, "initialize_copy", producer_init_copy, 1);
 
 	/* Producer.push_single(msg) */
-	rb_define_method(c_producer, "push_single", producer_push_single, 3);
+	rb_define_method(c_producer, "push_single", producer_push_single, 4);
 
 	/* Producer.tick */
 	rb_define_method(c_producer, "tick", producer_tick, 1);

data/ext/hermann/rdcrc32.h

@@ -0,0 +1,103 @@
+/**
+ * \file rdcrc32.h
+ * Functions and types for CRC checks.
+ *
+ * Generated on Tue May  8 17:36:59 2012,
+ * by pycrc v0.7.10, http://www.tty1.net/pycrc/
+ *
+ * NOTE: Contains librd modifications:
+ *       - rd_crc32() helper.
+ *       - __RDCRC32___H__ define (was missing the '32' part).
+ *
+ * using the configuration:
+ *    Width        = 32
+ *    Poly         = 0x04c11db7
+ *    XorIn        = 0xffffffff
+ *    ReflectIn    = True
+ *    XorOut       = 0xffffffff
+ *    ReflectOut   = True
+ *    Algorithm    = table-driven
+ *****************************************************************************/
+#ifndef __RDCRC32___H__
+#define __RDCRC32___H__
+
+#include <stdlib.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/**
+ * The definition of the used algorithm.
+ *****************************************************************************/
+#define CRC_ALGO_TABLE_DRIVEN 1
+
+
+/**
+ * The type of the CRC values.
+ *
+ * This type must be big enough to contain at least 32 bits.
+ *****************************************************************************/
+typedef uint32_t rd_crc32_t;
+
+
+/**
+ * Reflect all bits of a \a data word of \a data_len bytes.
+ *
+ * \param data         The data word to be reflected.
+ * \param data_len     The width of \a data expressed in number of bits.
+ * \return             The reflected data.
+ *****************************************************************************/
+rd_crc32_t rd_crc32_reflect(rd_crc32_t data, size_t data_len);
+
+
+/**
+ * Calculate the initial crc value.
+ *
+ * \return     The initial crc value.
+ *****************************************************************************/
+static inline rd_crc32_t rd_crc32_init(void)
+{
+    return 0xffffffff;
+}
+
+
+/**
+ * Update the crc value with new data.
+ *
+ * \param crc      The current crc value.
+ * \param data     Pointer to a buffer of \a data_len bytes.
+ * \param data_len Number of bytes in the \a data buffer.
+ * \return         The updated crc value.
+ *****************************************************************************/
+rd_crc32_t rd_crc32_update(rd_crc32_t crc, const unsigned char *data, size_t data_len);
+
+
+/**
+ * Calculate the final crc value.
+ *
+ * \param crc  The current crc value.
+ * \return     The final crc value.
+ *****************************************************************************/
+static inline rd_crc32_t rd_crc32_finalize(rd_crc32_t crc)
+{
+    return crc ^ 0xffffffff;
+}
+
+
+/**
+ * Wrapper for performing CRC32 on the provided buffer.
+ */
+static inline rd_crc32_t rd_crc32 (const char *data, size_t data_len) {
+  return rd_crc32_finalize(rd_crc32_update(rd_crc32_init(),
+             (const unsigned char *)data,
+             data_len));
+}
+
+#ifdef __cplusplus
+}           /* closing brace for extern "C" */
+#endif
+
+#endif      /* __RDCRC32___H__ */

data/lib/hermann/producer.rb

@@ -63,8 +63,7 @@ module Hermann
       end
 
       if Hermann.jruby?
-        key = opts.has_key?(:partition_key) ? opts[:partition_key].to_java : nil
-        result = @internal.push_single(value, topic, key)
+        result = @internal.push_single(value, topic, opts[:partition_key], nil)
         unless result.nil?
           @children << result
         end
@@ -76,7 +75,7 @@ module Hermann
         # librdkafka callback queue overflow
         tick_reactor
         result = create_result
-        @internal.push_single(value, topic, result)
+        @internal.push_single(value, topic, opts[:partition_key].to_s, result)
       end
 
       return result

data/lib/hermann/provider/java_producer.rb

@@ -43,7 +43,8 @@ module Hermann
       # @return +Concurrent::Promise+ Representa a promise to send the
       #   data to the kafka broker.  Upon execution the Promise's status
       #   will be set
-      def push_single(msg, topic, key)
+      def push_single(msg, topic, key, _)
+        key = key && key.to_java
         Concurrent::Promise.execute {
           data = ProducerUtil::KeyedMessage.new(topic, nil, key, msg.to_java_bytes)
           begin

data/lib/hermann/version.rb

@@ -1,3 +1,3 @@
 module Hermann
-  VERSION = '0.26.0'
+  VERSION = '0.26.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hermann
 version: !ruby/object:Gem::Version
-  version: 0.26.0.0
+  version: 0.26.1.0
 platform: java
 authors:
 - R. Tyler Croy
@@ -10,12 +10,12 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-09-08 00:00:00.000000000 Z
+date: 2015-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.8.2
   name: json
@@ -23,13 +23,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.8.2
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.4
   name: thread_safe
@@ -37,13 +37,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.4
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.7.0
   name: concurrent-ruby
@@ -51,16 +51,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.7.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.1.10
   name: jar-dependencies
@@ -68,10 +68,10 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.1.10
 description: Ruby gem for talking to Kafka
@@ -87,6 +87,7 @@ files:
 - ext/hermann/extconf.rb
 - ext/hermann/hermann_rdkafka.c
 - ext/hermann/hermann_rdkafka.h
+- ext/hermann/rdcrc32.h
 - lib/hermann.rb
 - lib/hermann/consumer.rb
 - lib/hermann/discovery/metadata.rb
@@ -110,21 +111,20 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
 - jar org.apache.kafka:kafka_2.10, ~>0.8.1.1
 - jar log4j:log4j, ~>1.2.16
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 3
 summary: A Kafka consumer/producer gem supporting both MRI and JRuby
 test_files: []
-has_rdoc: