hermann 0.22.0-java → 0.23.0.236-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e66cc3de18d3ed620a691c0e308068f471bb89b6
-  data.tar.gz: 9c1c1171bec630a9cd8ee843e4299bd5d016707a
+  metadata.gz: 72692497af5d422995313c36700af4b0cb832411
+  data.tar.gz: da5d00bd780a496573cb007a32d6b1e5bea2a775
 SHA512:
-  metadata.gz: dcb43908793f2254b2839c59cca84f9e74fc11ae8f06739f984398caf550b223c751891376a9af786963a93c97a1814e9d96db1f7a24095e6b02a87c87c71078
-  data.tar.gz: 80558b4a81be616687d04ce29e8106c710621db6a4cacb71f1cde3f2a1d9761c785e209fb4dc30738d4da9aaec6d84649d07ba9684fc2bef9745288db03c1d99
+  metadata.gz: 2bf5ce855f661db9117bf622de88de8d45233f07c1d3c21282b73052c640cee7d0d86212946301fe21865623955ffa3cb47312927022021e08c66a2b3625acf2
+  data.tar.gz: de6b0f7ea494ecc23d1cc95d0e12516aba88f38aa08ed99613ef008dc753bc805147aa7f91768ce90b462eafea9eae4661340f4d9d1185532092e62a4457428d

data/ext/hermann/extconf.rb

@@ -146,4 +146,7 @@ dir_config('rdkafka', HEADER_DIRS, LIB_DIRS)
 #   <http://blog.zachallett.com/howto-ruby-c-extension-with-a-static-library>
 $LOCAL_LIBS << File.join(librdkafka.path, 'lib', 'librdkafka.a')
 
+have_header('ruby/thread.h')
+have_func('rb_thread_call_without_gvl')
+
 create_makefile('hermann/hermann_lib')

data/ext/hermann/hermann_lib.c

@@ -33,6 +33,11 @@
 
 #include "hermann_lib.h"
 
+
+/* how long to let librdkafka block on the socket before returning back to the interpreter.
+ * essentially defines how long we wait before consumer_consume_stop_callback() can fire */
+#define CONSUMER_RECVMSG_TIMEOUT_MS 100
+
 /**
  * Convenience function
  *
@@ -204,22 +209,16 @@ static void hexdump(FILE *fp,
  * @param rkmessage	rd_kafka_message_t* the message
  * @param opaque	   void*   opaque context
  */
-static void msg_consume(rd_kafka_message_t *rkmessage,
-						void *opaque) {
-
-	HermannInstanceConfig* cfg;
-
-	cfg = (HermannInstanceConfig*)opaque;
-
+static void msg_consume(rd_kafka_message_t *rkmessage, HermannInstanceConfig *cfg) {
 	if (rkmessage->err) {
 		if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
-			fprintf(stderr,
-				"%% Consumer reached end of %s [%"PRId32"] "
-				   "message queue at offset %"PRId64"\n",
-				   rd_kafka_topic_name(rkmessage->rkt),
-				   rkmessage->partition, rkmessage->offset);
-
 			if (cfg->exit_eof) {
+				fprintf(stderr,
+						"%% Consumer reached end of %s [%"PRId32"] "
+						"message queue at offset %"PRId64"\n",
+						rd_kafka_topic_name(rkmessage->rkt),
+						rkmessage->partition, rkmessage->offset);
+
 				cfg->run = 0;
 			}
 
@@ -341,7 +340,7 @@ void consumer_init_kafka(HermannInstanceConfig* config) {
 
 // Ruby gem extensions
 
-#ifdef RB_THREAD_BLOCKING_REGION
+#if defined(RB_THREAD_BLOCKING_REGION) || defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL)
 /* NOTE: We only need this method defined if RB_THREAD_BLOCKING_REGION is
  * defined, otherwise it's unused
  */
@@ -360,21 +359,64 @@ static void consumer_consume_stop_callback(void *ptr) {
 #endif
 
 /**
- * Loop on a timeout to receive messages from Kafka.  When the consumer_consume_stop_callback is invoked by Ruby,
- * we'll break out of our loop and return.
+ * consumer_recv_msg
+ *
+ * Consume a single message from the kafka stream.  The only function that should be invoked
+ * without the GVL held.
+ *
+ * @param   HermannInstanceConfig* The hermann configuration for this consumer
+ *
  */
-void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
 
+static void *consumer_recv_msg(void *ptr)
+{
+	rd_kafka_message_t *ret;
+	HermannInstanceConfig *consumerConfig = (HermannInstanceConfig *) ptr;
+
+	ret = rd_kafka_consume(consumerConfig->rkt, consumerConfig->partition, CONSUMER_RECVMSG_TIMEOUT_MS);
+
+	if ( ret == NULL ) {
+		if ( errno != ETIMEDOUT )
+			fprintf(stderr, "%% Error: %s\n", rd_kafka_err2str( rd_kafka_errno2err(errno)));
+	}
+
+	return (void *) ret;
+}
+
+/**
+ * consumer_consume_loop
+ *
+ * A timeout-interrupted loop in which we drop the GVL and attemptto receive
+ * messages from Kafka.  We'll check every  CONSUMER_RECVMSG_TIMEOUT_MS, or
+ * after every message, to see if the ruby interpreter wants us to exit the
+ * loop.
+ *
+ * @param   HermannInstanceConfig* The hermann configuration for this consumer
+ */
+
+static void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
+	rd_kafka_message_t *msg;
 	TRACER("\n");
 
 	while (consumerConfig->run) {
-		if (rd_kafka_consume_callback(consumerConfig->rkt, consumerConfig->partition,
-										  1000/*timeout*/,
-										  msg_consume,
-										  consumerConfig) < 0) {
-			fprintf(stderr, "%% Error: %s\n", rd_kafka_err2str( rd_kafka_errno2err(errno)));
-		}
+#ifdef RB_THREAD_BLOCKING_REGION
+		msg = rb_thread_blocking_region(consumer_recv_msg,
+				consumerConfig,
+				consumer_consume_stop_callback,
+				consumerConfig);
+#elif HAVE_RB_THREAD_CALL_WITHOUT_GVL
+		msg = rb_thread_call_without_gvl(consumer_recv_msg,
+				consumerConfig,
+				consumer_consume_stop_callback,
+				consumerConfig);
+#else
+		msg = consumer_recv_msg(consumerConfig);
+#endif
 
+		if ( msg ) {
+			msg_consume(msg, consumerConfig);
+			rd_kafka_message_destroy(msg);
+		}
 	}
 }
 
@@ -412,23 +454,7 @@ static VALUE consumer_consume(VALUE self, VALUE topic) {
 		return Qnil;
 	}
 
-#ifdef RB_THREAD_BLOCKING_REGION
-	/** The consumer will listen for incoming messages in a loop, timing out and checking the consumerConfig->run
-	 *  flag every second.
-	 *
-	 *  Call rb_thread_blocking_region to release the GVM lock and allow Ruby to amuse itself while we wait on
-	 *  IO from Kafka.
-	 *
-	 *  If Ruby needs to interrupt the consumer loop, the stop callback will be invoked and the loop should exit.
-	 */
-	rb_thread_blocking_region(consumer_consume_loop,
-							  consumerConfig,
-							  consumer_consume_stop_callback,
-							  consumerConfig);
-#else
-	consumer_consume_loop(consumerConfig);
-#endif
-
+  consumer_consume_loop(consumerConfig);
 
 	/* Stop consuming */
 	rd_kafka_consume_stop(consumerConfig->rkt, consumerConfig->partition);
@@ -669,7 +695,7 @@ static VALUE producer_connect(VALUE self, VALUE timeout) {
 								   &data,
 								   timeout_ms);
 	TRACER("err: %s (%i)\n", rd_kafka_err2str(err), err);
-	
+
 	if (RD_KAFKA_RESP_ERR_NO_ERROR == err) {
 		TRACER("brokers: %i, topics: %i\n",
 				data->broker_cnt,

data/ext/hermann/hermann_lib.h

@@ -32,6 +32,10 @@
 
 #include <ruby.h>
 
+#ifdef HAVE_RUBY_THREAD_H
+#include <ruby/thread.h>
+#endif
+
 #include <ctype.h>
 #include <signal.h>
 #include <string.h>

data/lib/hermann/consumer.rb

@@ -10,30 +10,27 @@ module Hermann
   # Hermann::Consumer provides a simple consumer API which is only safe to be
   # executed in a single thread
   class Consumer
-    attr_reader :topic, :brokers, :partition, :internal
+    attr_reader :topic, :internal
 
 
     # Instantiate Consumer
     #
     # @params [String] kafka topic
-    #
-    # @params [String] group ID
-    #
-    # @params [String] comma separated zookeeper list
-    #
     # @params [Hash] options for Consumer
-    # @option opts [String] :brokers   (for MRI) Comma separated list of brokers
-    # @option opts [Integer] :partition (for MRI) The kafka partition
-    def initialize(topic, groupId, zookeepers, opts={})
+    # @option opts [String]  :brokers    (for MRI) Comma separated list of brokers
+    # @option opts [Integer] :partition  (for MRI) The kafka partition
+    # @option opts [Integer] :zookeepers (for jruby) list of zookeeper servers
+    # @option opts [Integer] :group_id   (for jruby) client group_id
+    #
+    def initialize(topic, opts = {})
       @topic = topic
-      @brokers = brokers
-      @partition = partition
-
       if Hermann.jruby?
-        @internal = Hermann::Provider::JavaSimpleConsumer.new(zookeepers, groupId, topic, opts)
+        zookeepers, group_id = require_values_at(opts, :zookeepers, :group_id)
+
+        @internal = Hermann::Provider::JavaSimpleConsumer.new(zookeepers, group_id, topic, opts)
       else
-        brokers   = opts.delete(:brokers)
-        partition = opts.delete(:partition)
+        brokers, partition = require_values_at(opts, :brokers, :partition)
+
         @internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
       end
     end
@@ -51,5 +48,12 @@ module Hermann
         #no op
       end
     end
+
+    def require_values_at(opts, *args)
+      args.map do |a|
+        raise "Please provide :#{a} option!" unless opts[a]
+        opts.delete(a)
+      end
+    end
   end
 end

data/lib/hermann/version.rb

@@ -1,3 +1,3 @@
 module Hermann
-  VERSION = '0.22.0'
+  VERSION = '0.23.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hermann
 version: !ruby/object:Gem::Version
-  version: 0.22.0
+  version: 0.23.0.236
 platform: java
 authors:
 - R. Tyler Croy
@@ -10,50 +10,50 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-07 00:00:00.000000000 Z
+date: 2015-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.7.0
-  name: concurrent-ruby
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: thread_safe
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.7.0
-- !ruby/object:Gem::Dependency
+        version: 0.3.4
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.4
-  name: thread_safe
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jar-dependencies
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.3.4
-- !ruby/object:Gem::Dependency
+        version: 0.1.9
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.9
-  name: jar-dependencies
   prerelease: false
   type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.1.9
 description: Ruby gem for talking to Kafka
 email:
 - rtyler.croy@lookout.com