hermann 0.24.0.0 → 0.24.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 391d3fb8e74708554ec391bdc229766037b16002
-  data.tar.gz: 693ef0723e454299972e28290966a6cfa64ca42a
+  metadata.gz: 7ab2b9a3c3699c6d765e930686fd0fb08df9e244
+  data.tar.gz: 36e43ca4b80e8428760e80b3ba65d6ff35a43232
 SHA512:
-  metadata.gz: 134ec914a11f0820cc177f0ceccf8af360e4863f2866985fe0c493e05ac321e294e296399ec1d378507e0e3e4cee5ae7a35ae038470cb3eb291630e9fe975652
-  data.tar.gz: bd47334161757bb87ad7dc1faefa884a9a6b8d43de4a21eebfb214c538860b74ae7ae6ce5e20a7f7e2ef363e338f351781ad6c33f69f10a0fa4796002d57059a
+  metadata.gz: 6ba78bfd1051a7e9f13b29d2816bbb7ac973968c081743dcf15fd85a4da7bfdc2c12c5bd7bbef00a13cc089c0489e78c1447238ce4aec59263b71bbf7db67881
+  data.tar.gz: 7d4d31a0ee898b78fe921778b788d5a661b7d5d6630ecd95053692595f3cf37cb754399fbf30f5221f9ba481b6be0ab5eb23ca226c09a7f38d8d7695f3c413c9

data/ext/hermann/extconf.rb

@@ -147,6 +147,7 @@ $LOCAL_LIBS << File.join(librdkafka.path, 'lib', 'librdkafka.a')
 
 have_header('ruby/thread.h')
 have_header('ruby/intern.h')
+have_header('ruby/version.h')
 have_func('rb_thread_blocking_region')
 have_func('rb_thread_call_without_gvl')
 

data/ext/hermann/hermann_lib.c

@@ -33,6 +33,9 @@
 
 #include "hermann_lib.h"
 
+#ifdef HAVE_RUBY_VERSION_H
+#include <ruby/version.h>
+#endif
 
 /* how long to let librdkafka block on the socket before returning back to the interpreter.
  * essentially defines how long we wait before consumer_consume_stop_callback() can fire */
@@ -120,7 +123,7 @@ static void msg_delivered(rd_kafka_t *rk,
 		/* call back into our Hermann::Result if it exists, discarding the
 		* return value
 		*/
-		if (NULL != push_ctx->result) {
+		if (NULL != (void *)push_ctx->result) {
 			rb_funcall(push_ctx->result,
 						hermann_result_fulfill_method,
 						2,
@@ -153,12 +156,15 @@ static int32_t producer_partitioner_callback(const rd_kafka_topic_t *rkt,
 											 void *msg_opaque) {
 	/* Pick a random partition */
 	int retry = 0;
+	int32_t partition = RD_KAFKA_PARTITION_UA;
+
 	for (; retry < partition_cnt; retry++) {
-		int32_t partition = rand() % partition_cnt;
+		partition = rand() % partition_cnt;
 		if (rd_kafka_topic_partition_available(rkt, partition)) {
 			break; /* this one will do */
 		}
 	}
+	return partition;
 }
 
 /**
@@ -259,6 +265,7 @@ static void msg_consume(rd_kafka_message_t *rkmessage, HermannInstanceConfig *cf
 	// Yield the data to the Consumer's block
 	if (rb_block_given_p()) {
 		VALUE value = rb_str_new((char *)rkmessage->payload, rkmessage->len);
+		rd_kafka_message_destroy(rkmessage);
 		rb_yield(value);
 	}
 	else {
@@ -388,15 +395,19 @@ static void *consumer_recv_msg(void *ptr)
  * after every message, to see if the ruby interpreter wants us to exit the
  * loop.
  *
- * @param   HermannInstanceConfig* The hermann configuration for this consumer
+ * @param   self The consumer instance
  */
 
-static void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
+static VALUE consumer_consume_loop(VALUE self) {
+	HermannInstanceConfig* consumerConfig;
 	rd_kafka_message_t *msg;
+
+	Data_Get_Struct(self, HermannInstanceConfig, consumerConfig);
+
 	TRACER("\n");
 
 	while (consumerConfig->run) {
-#ifdef HAVE_RB_THREAD_BLOCKING_REGION
+#if HAVE_RB_THREAD_BLOCKING_REGION && RUBY_API_VERSION_MAJOR < 2
 		msg = (rd_kafka_message_t *) rb_thread_blocking_region((rb_blocking_function_t *) consumer_recv_msg,
 				consumerConfig,
 				consumer_consume_stop_callback,
@@ -412,9 +423,24 @@ static void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
 
 		if ( msg ) {
 			msg_consume(msg, consumerConfig);
-			rd_kafka_message_destroy(msg);
 		}
 	}
+
+	return Qnil;
+}
+
+
+/**
+ * consumer_consume_loop_stop
+ *
+ * called when we're done with the .consume() loop.  lets rdkafa cleanup some internal structures
+ */
+static VALUE consumer_consume_loop_stop(VALUE self) {
+	HermannInstanceConfig* consumerConfig;
+	Data_Get_Struct(self, HermannInstanceConfig, consumerConfig);
+
+	rd_kafka_consume_stop(consumerConfig->rkt, consumerConfig->partition);
+  return Qnil;
 }
 
 /**
@@ -446,17 +472,12 @@ static VALUE consumer_consume(VALUE self, VALUE topic) {
 	if (rd_kafka_consume_start(consumerConfig->rkt, consumerConfig->partition, consumerConfig->start_offset) == -1) {
 		fprintf(stderr, "%% Failed to start consuming: %s\n",
 			rd_kafka_err2str(rd_kafka_errno2err(errno)));
-		rb_raise(rb_eRuntimeError,
+		rb_raise(rb_eRuntimeError, "%s",
 				rd_kafka_err2str(rd_kafka_errno2err(errno)));
 		return Qnil;
 	}
 
-  consumer_consume_loop(consumerConfig);
-
-	/* Stop consuming */
-	rd_kafka_consume_stop(consumerConfig->rkt, consumerConfig->partition);
-
-	return Qnil;
+	return rb_ensure(consumer_consume_loop, self, consumer_consume_loop_stop, self);
 }
 
 
@@ -575,7 +596,7 @@ static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE
 	Data_Get_Struct(self, HermannInstanceConfig, producerConfig);
 
 	delivery_ctx->producer = producerConfig;
-	delivery_ctx->result = NULL;
+	delivery_ctx->result = (VALUE) NULL;
 
 	TRACER("producerConfig: %p\n", producerConfig);
 
@@ -666,19 +687,56 @@ static VALUE producer_tick(VALUE self, VALUE timeout) {
 	events = rd_kafka_poll(conf->rk, timeout_ms);
 
 	if (conf->isErrored) {
-		rb_raise(rb_eStandardError, conf->error);
+		rb_raise(rb_eStandardError, "%s", conf->error);
 	}
 
 	return rb_int_new(events);
 }
 
+/*
+ *  producer_metadata_request_nogvl
+ *
+ *  call rd_kafka_metadata without the GVL held.  Note that rd_kafka_metadata is not interruptible,
+ *  so in case of interrupt the thread will not respond until timeout_ms is reached.
+ *
+ *  rd_kafka_metadata will fill in the ctx->data pointer on success
+ *
+ *  @param  ptr   void*  the hermann_metadata_ctx_t
+ */
+
+static void *producer_metadata_request_nogvl(void *ptr)
+{
+	hermann_metadata_ctx_t *ctx = (hermann_metadata_ctx_t*)ptr;
+
+	return (void *) rd_kafka_metadata(ctx->rk,
+			ctx->topic ? 0 : 1,
+			ctx->topic,
+			(const struct rd_kafka_metadata **) &(ctx->data),
+			ctx->timeout_ms);
+}
+
+
+static int producer_metadata_request(hermann_metadata_ctx_t *ctx)
+{
+	int err;
+
+#if HAVE_RB_THREAD_BLOCKING_REGION && RUBY_API_VERSION_MAJOR < 2
+	err = (int) rb_thread_blocking_region((rb_blocking_function_t *) producer_metadata_request_nogvl, ctx,
+				NULL, NULL);
+#elif HAVE_RB_THREAD_CALL_WITHOUT_GVL
+	err = (int) rb_thread_call_without_gvl(producer_metadata_request_nogvl, ctx, NULL, NULL);
+#else
+	err = (int) producer_metadata_request_nogvl(ctx);
+#endif
+
+	return err;
+}
 
 static VALUE producer_connect(VALUE self, VALUE timeout) {
 	HermannInstanceConfig *producerConfig;
 	rd_kafka_resp_err_t err;
 	VALUE result = Qfalse;
-	int timeout_ms = rb_num2int(timeout);
-	struct rd_kafka_metadata *data = NULL;
+	hermann_metadata_ctx_t md_context;
 
 	Data_Get_Struct(self, HermannInstanceConfig, producerConfig);
 
@@ -686,17 +744,19 @@ static VALUE producer_connect(VALUE self, VALUE timeout) {
 		producer_init_kafka(self, producerConfig);
 	}
 
-	err = rd_kafka_metadata(producerConfig->rk,
-									0,
-								   producerConfig->rkt,
-								   &data,
-								   timeout_ms);
+	md_context.rk = producerConfig->rk;
+	md_context.topic = NULL;
+	md_context.data = NULL;
+	md_context.timeout_ms = rb_num2int(timeout);
+
+	err = producer_metadata_request(&md_context);
+
 	TRACER("err: %s (%i)\n", rd_kafka_err2str(err), err);
 
 	if (RD_KAFKA_RESP_ERR_NO_ERROR == err) {
 		TRACER("brokers: %i, topics: %i\n",
-				data->broker_cnt,
-				data->topic_cnt);
+				md_context.data->broker_cnt,
+				md_context.data->topic_cnt);
 		producerConfig->isConnected = 1;
 		result = Qtrue;
 	}
@@ -704,11 +764,118 @@ static VALUE producer_connect(VALUE self, VALUE timeout) {
 		producerConfig->isErrored = err;
 	}
 
-	rd_kafka_metadata_destroy(data);
+	if ( md_context.data )
+		rd_kafka_metadata_destroy(md_context.data);
 
 	return result;
 }
 
+/*
+ *  producer_metadata_make_hash
+ *
+ *  transform the rd_kafka_metadata structure into a ruby hash.  eg:
+ *  { :brokers => [ {:id=>0, :host=>"172.20.10.3", :port=>9092} ],
+ *    :topics => { "maxwell" => [ {:id=>0, :leader_id=>0, :replica_ids=>[0], :isr_ids=>[0]}]} }
+ *
+ *  @param  data  struct rd_kafka_metadata* data returned from rd_kafka_metadata
+ */
+
+static VALUE producer_metadata_make_hash(struct rd_kafka_metadata *data)
+{
+	int i, j, k;
+	VALUE broker_hash, topic_hash, partition_ary, partition_hash, partition_replica_ary, partition_isr_ary;
+	VALUE hash = rb_hash_new();
+	VALUE brokers = rb_ary_new2(data->broker_cnt);
+	VALUE topics =  rb_hash_new();
+
+	for ( i = 0; i < data->broker_cnt; i++ ) {
+		broker_hash = rb_hash_new();
+		rb_hash_aset(broker_hash, ID2SYM(rb_intern("id")),   INT2FIX(data->brokers[i].id));
+		rb_hash_aset(broker_hash, ID2SYM(rb_intern("host")), rb_str_new2(data->brokers[i].host));
+		rb_hash_aset(broker_hash, ID2SYM(rb_intern("port")), INT2FIX(data->brokers[i].port));
+		rb_ary_push(brokers, broker_hash);
+	}
+
+	for ( i = 0; i < data->topic_cnt; i++ ) {
+		partition_ary = rb_ary_new2(data->topics[i].partition_cnt);
+
+		for ( j = 0 ; j < data->topics[i].partition_cnt ; j++ ) {
+			VALUE partition_hash = rb_hash_new();
+			rd_kafka_metadata_partition_t *partition = &(data->topics[i].partitions[j]);
+
+			/* id => 1, leader_id => 0 */
+			rb_hash_aset(partition_hash, ID2SYM(rb_intern("id")), INT2FIX(partition->id));
+			rb_hash_aset(partition_hash, ID2SYM(rb_intern("leader_id")), INT2FIX(partition->leader));
+
+			/* replica_ids => [1, 0] */
+			partition_replica_ary = rb_ary_new2(partition->replica_cnt);
+			for ( k = 0 ; k < partition->replica_cnt ; k++ ) {
+				rb_ary_push(partition_replica_ary, INT2FIX(partition->replicas[k]));
+			}
+			rb_hash_aset(partition_hash, ID2SYM(rb_intern("replica_ids")), partition_replica_ary);
+
+			/* isr_ids => [1, 0] */
+			partition_isr_ary = rb_ary_new2(partition->isr_cnt);
+			for ( k = 0 ; k < partition->isr_cnt ; k++ ) {
+				rb_ary_push(partition_isr_ary, INT2FIX(partition->isrs[k]));
+			}
+			rb_hash_aset(partition_hash, ID2SYM(rb_intern("isr_ids")), partition_isr_ary);
+
+			rb_ary_push(partition_ary, partition_hash);
+		}
+
+		rb_hash_aset(topics, rb_str_new2(data->topics[i].topic), partition_ary);
+	}
+
+	rb_hash_aset(hash, ID2SYM(rb_intern("brokers")), brokers);
+	rb_hash_aset(hash, ID2SYM(rb_intern("topics")), topics);
+	return hash;
+}
+
+/*
+ *  producer_metadata
+ *
+ *  make a metadata request to the kafka server, returning a hash
+ *  containing a list of brokers and topics.
+ *
+ *  @param  data  struct rd_kafka_metadata* data returned from rd_kafka_metadata
+ */
+
+static VALUE producer_metadata(VALUE self, VALUE topicStr, VALUE timeout) {
+	HermannInstanceConfig *producerConfig;
+	rd_kafka_resp_err_t err;
+	hermann_metadata_ctx_t md_context;
+	VALUE result;
+
+	Data_Get_Struct(self, HermannInstanceConfig, producerConfig);
+
+	if (!producerConfig->isInitialized) {
+		producer_init_kafka(self, producerConfig);
+	}
+
+	md_context.rk = producerConfig->rk;
+	md_context.timeout_ms = rb_num2int(timeout);
+
+	if ( !NIL_P(topicStr) ) {
+		Check_Type(topicStr, T_STRING);
+		md_context.topic = rd_kafka_topic_new(producerConfig->rk, StringValuePtr(topicStr), NULL);
+	} else {
+		md_context.topic = NULL;
+	}
+
+	err = producer_metadata_request(&md_context);
+
+	if ( err != RD_KAFKA_RESP_ERR_NO_ERROR ) {
+		// annoyingly, this is always a timeout error -- the rest rdkafka just jams onto STDERR
+		rb_raise( rb_eRuntimeError, "%s", rd_kafka_err2str(err) );
+	} else {
+		result = producer_metadata_make_hash(md_context.data);
+		rd_kafka_metadata_destroy(md_context.data);
+		return result;
+	}
+
+}
+
 static VALUE producer_is_connected(VALUE self) {
 	HermannInstanceConfig *producerConfig;
 
@@ -1076,4 +1243,7 @@ void Init_hermann_lib() {
 
 	/* Producer.connect */
 	rb_define_method(c_producer, "connect", producer_connect, 1);
+
+	/* Producer.metadata */
+	rb_define_method(c_producer, "metadata", producer_metadata, 2);
 }

data/ext/hermann/hermann_lib.h

@@ -113,4 +113,11 @@ typedef struct {
 	VALUE result;
 } hermann_push_ctx_t;
 
+typedef struct {
+	rd_kafka_t *rk;
+	rd_kafka_topic_t *topic;
+	struct rd_kafka_metadata *data;
+	int timeout_ms;
+} hermann_metadata_ctx_t;
+
 #endif

data/lib/hermann/discovery/metadata.rb

@@ -0,0 +1,77 @@
+require 'hermann_lib'
+require 'hermann/consumer'
+
+module Hermann
+  module Discovery
+    class Metadata
+      Broker    = Struct.new(:id, :host, :port) do
+        def to_s
+          "#{host}:#{port}"
+        end
+      end
+      Topic     = Struct.new(:name, :partitions)
+
+      Partition = Struct.new(:id, :leader, :replicas, :insync_replicas, :topic_name) do
+        def consumer(offset=:end)
+          Hermann::Consumer.new(topic_name, brokers: ([leader] + replicas).join(','), partition: id, offset: offset)
+        end
+      end
+
+      DEFAULT_TIMEOUT_MS = 2_000
+      def initialize(brokers, options = {})
+        raise "this is an MRI api only!" if Hermann.jruby?
+        @internal = Hermann::Lib::Producer.new(brokers)
+        @timeout = options[:timeout] || DEFAULT_TIMEOUT_MS
+      end
+
+      #
+      # @internal.metadata returns:
+      # {:brokers => [{:id=>3, :host=>"kafka3.alpha4.sac1.zdsys.com", :port=>9092}],
+      #  :topics  => {"testtopic"=>[{:id=>0, :leader_id=>3, :replica_ids=>[3, 1],  :isr_ids=>[3, 1]}}}
+      #
+      def brokers
+        brokers_from_metadata(@internal.metadata(nil, @timeout))
+      end
+
+      def topic(t)
+        get_topics(t)[t]
+      end
+
+      def topics
+        get_topics
+      end
+
+      private
+
+      def get_topics(filter_topics = nil)
+        md = @internal.metadata(filter_topics, @timeout)
+
+        broker_hash = brokers_from_metadata(md).inject({}) do |h, broker|
+          h[broker.id] = broker
+          h
+        end
+
+        md[:topics].inject({}) do |topic_hash, arr|
+          topic_name, raw_partitions = *arr
+          partitions = raw_partitions.map do |p|
+            leader       = broker_hash[p[:leader_id]]
+            all_replicas = p[:replica_ids].map { |i| broker_hash[i] }
+            isr_replicas = p[:isr_ids].map { |i| broker_hash[i] }
+            Partition.new(p[:id], leader, all_replicas, isr_replicas, topic_name)
+          end
+
+          topic_hash[topic_name] = Topic.new(topic_name, partitions)
+          topic_hash
+        end
+      end
+
+
+      def brokers_from_metadata(md)
+        md[:brokers].map do |h|
+          Broker.new(h[:id], h[:host], h[:port])
+        end
+      end
+
+    end
+  end
+end

data/lib/hermann/version.rb

@@ -1,3 +1,3 @@
 module Hermann
-  VERSION = '0.24.0'
+  VERSION = '0.24.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hermann
 version: !ruby/object:Gem::Version
-  version: 0.24.0.0
+  version: 0.24.1.0
 platform: ruby
 authors:
 - R. Tyler Croy
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-15 00:00:00.000000000 Z
+date: 2015-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -70,6 +70,7 @@ files:
 - ext/hermann/hermann_lib.h
 - lib/hermann.rb
 - lib/hermann/consumer.rb
+- lib/hermann/discovery/metadata.rb
 - lib/hermann/discovery/zookeeper.rb
 - lib/hermann/errors.rb
 - lib/hermann/java.rb