hermann 0.17.0 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 --- 
 SHA1: 
-  metadata.gz: b08abedfd6c15b1c7c1aa5dfe66315501c212512
-  data.tar.gz: 3bf7306d482a260fd73004e4d8087cd5f3a1f063
+  metadata.gz: 92486b735dc5e9e88b0243fabc35340cd3c3a117
+  data.tar.gz: aa5b9762eb50d57943a1c6bf1a78d84abc6c5055
 SHA512: 
-  metadata.gz: 901a4eecf276229c561186f5be901b6ed87313d35d6878365621ee0302328870f24fe02661620d633f4e55462aaf953a8036c3da88d5d6e6d66c7e71e9e6a9d4
-  data.tar.gz: 300db74c52df5be4f01078b6098c9a0cd528da811c4681e2fb0dde337c7b5b9bfe1cbe645769f9c616adbc9cfe5a841d9413f2d51208518e2dd1a20a4fce57cf
+  metadata.gz: 52e2b6523b9d0dd60e14a90e73babca4bb07e8ee2a72d2b3eecf799ede905e61813765e9d32d25cfc5194558f4551a914218bb8537e11debea2a13ab629f346e
+  data.tar.gz: 9847e3e907dfcba53523e7f019041a8c73f046efc8f5d1b04afbcec148e9b5d0f72b2aa2fc6154af3db11799ddc55e7646cc3b7687df0bcde43eefbf79259614

data/ext/hermann/extconf.rb

@@ -44,6 +44,81 @@ class RdKafkaRecipe < MiniPortile
       raise 'Checksum error!'
     end
   end
+
+  def download_file_http(url, full_path, count = 3)
+    filename = File.basename(full_path)
+    uri = URI.parse(url)
+
+    if ENV['http_proxy']
+      _, userinfo, p_host, p_port = URI.split(ENV['http_proxy'])
+      proxy_user, proxy_pass = userinfo.split(/:/) if userinfo
+      http = Net::HTTP.new(uri.host, uri.port, p_host, p_port, proxy_user, proxy_pass)
+    else
+      http = Net::HTTP.new(uri.host, uri.port)
+
+      if URI::HTTPS === uri
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+
+        store = OpenSSL::X509::Store.new
+
+        # Auto-include system-provided certificates
+        store.set_default_paths
+
+        if ENV.has_key?("SSL_CERT_FILE") && File.exist?(ENV["SSL_CERT_FILE"])
+          store.add_file ENV["SSL_CERT_FILE"]
+        end
+
+        http.cert_store = store
+      end
+    end
+
+    message "Downloading #{filename} "
+    http.start do |h|
+      h.request_get(uri.path, 'Accept-Encoding' => 'identity') do |response|
+        case response
+        when Net::HTTPNotFound
+          output "404 - Not Found"
+          return false
+
+        when Net::HTTPClientError
+          output "Error: Client Error: #{response.inspect}"
+          return false
+
+        when Net::HTTPRedirection
+          raise "Too many redirections for the original URL, halting." if count <= 0
+          url = response["location"]
+          return download_file(url, full_path, count - 1)
+
+        when Net::HTTPOK
+          return with_tempfile(filename, full_path) do |temp_file|
+            size = 0
+            progress = 0
+            total = response.header["Content-Length"].to_i
+
+            if total == 0
+              # There are cases when apparently GitHub.com will return an empty
+              # content-length header, which means we can't really trust the
+              # response, so we'll treat it like a redirect
+              puts "Empty content-length header, retrying"
+              return download_file(url, full_path, count - 1)
+            end
+
+            response.read_body do |chunk|
+              temp_file << chunk
+              size += chunk.size
+              new_progress = (size * 100) / total
+              unless new_progress == progress
+                message "\rDownloading %s (%3d%%) " % [filename, new_progress]
+              end
+              progress = new_progress
+            end
+            output
+          end
+        end
+      end
+    end
+  end
 end
 ################################################################################
 

data/ext/hermann/hermann_lib.c

@@ -534,16 +534,19 @@ void producer_init_kafka(VALUE self, HermannInstanceConfig* config) {
  *
  *  @param  self	VALUE   the Ruby producer instance
  *  @param  message VALUE   the ruby String containing the outgoing message.
+ *  @param  topic   VALUE   the ruby String containing the topic to use for the
+ *							outgoing message.
  *  @param  result  VALUE   the Hermann::Result object to be fulfilled when the
  *		push completes
  */
-static VALUE producer_push_single(VALUE self, VALUE message, VALUE result) {
+static VALUE producer_push_single(VALUE self, VALUE message, VALUE topic, VALUE result) {
 
 	HermannInstanceConfig* producerConfig;
 	/* Context pointer, pointing to `result`, for the librdkafka delivery
 	 * callback
 	 */
 	hermann_push_ctx_t *delivery_ctx = (hermann_push_ctx_t *)malloc(sizeof(hermann_push_ctx_t));
+	rd_kafka_topic_t *rkt = NULL;
 
 	TRACER("self: %p, message: %p, result: %p)\n", self, message, result);
 
@@ -554,10 +557,9 @@ static VALUE producer_push_single(VALUE self, VALUE message, VALUE result) {
 
 	TRACER("producerConfig: %p\n", producerConfig);
 
-	if ((NULL == producerConfig->topic) ||
-		(0 == strlen(producerConfig->topic))) {
-		fprintf(stderr, "Topic is null!\n");
-		rb_raise(rb_eRuntimeError, "Topic cannot be empty");
+	if ((Qnil == topic) ||
+		(0 == RSTRING_LEN(topic))) {
+		rb_raise(rb_eArgError, "Topic cannot be empty");
 		return self;
 	}
 
@@ -567,6 +569,15 @@ static VALUE producer_push_single(VALUE self, VALUE message, VALUE result) {
 
 	TRACER("kafka initialized\n");
 
+	rkt = rd_kafka_topic_new(producerConfig->rk,
+								RSTRING_PTR(topic),
+								NULL);
+
+	if (NULL == rkt) {
+		rb_raise(rb_eRuntimeError, "Could not construct a topic structure");
+		return self;
+	}
+
 	/* Only pass result through if it's non-nil */
 	if (Qnil != result) {
 		delivery_ctx->result = result;
@@ -576,7 +587,7 @@ static VALUE producer_push_single(VALUE self, VALUE message, VALUE result) {
 	TRACER("rd_kafka_produce() message of %i bytes\n", RSTRING_LEN(message));
 
 	/* Send/Produce message. */
-	if (-1 == rd_kafka_produce(producerConfig->rkt,
+	if (-1 == rd_kafka_produce(rkt,
 						 producerConfig->partition,
 						 RD_KAFKA_MSG_F_COPY,
 						 RSTRING_PTR(message),
@@ -590,6 +601,10 @@ static VALUE producer_push_single(VALUE self, VALUE message, VALUE result) {
 		/* TODO: raise a Ruby exception here, requires a test though */
 	}
 
+	if (NULL != rkt) {
+		rd_kafka_topic_destroy(rkt);
+	}
+
 	TRACER("returning\n");
 
 	return self;
@@ -913,11 +928,9 @@ static VALUE producer_allocate(VALUE klass) {
  *  Set up the configuration context for the Producer instance
  *
  *  @param  self	VALUE   the Producer instance
- *  @param  topic   VALUE   the Ruby string naming the topic
  *  @param  brokers VALUE   a Ruby string containing host:port pairs separated by commas
  */
 static VALUE producer_initialize(VALUE self,
-								 VALUE topic,
 								 VALUE brokers) {
 
 	HermannInstanceConfig* producerConfig;
@@ -926,12 +939,9 @@ static VALUE producer_initialize(VALUE self,
 
 	TRACER("initialize Producer ruby object\n");
 
-
-	topicPtr = StringValuePtr(topic);
 	brokersPtr = StringValuePtr(brokers);
 	Data_Get_Struct(self, HermannInstanceConfig, producerConfig);
 
-	producerConfig->topic = topicPtr;
 	producerConfig->brokers = brokersPtr;
 	/** Using RD_KAFKA_PARTITION_UA specifies we want the partitioner callback to be called to determine the target
 	 *  partition
@@ -1011,11 +1021,11 @@ void Init_hermann_lib() {
 	rb_define_alloc_func(c_producer, producer_allocate);
 
 	/* Initialize */
-	rb_define_method(c_producer, "initialize", producer_initialize, 2);
+	rb_define_method(c_producer, "initialize", producer_initialize, 1);
 	rb_define_method(c_producer, "initialize_copy", producer_init_copy, 1);
 
 	/* Producer.push_single(msg) */
-	rb_define_method(c_producer, "push_single", producer_push_single, 2);
+	rb_define_method(c_producer, "push_single", producer_push_single, 3);
 
 	/* Producer.tick */
 	rb_define_method(c_producer, "tick", producer_tick, 1);

data/lib/hermann/discovery/zookeeper.rb

@@ -1,10 +1,11 @@
 require 'hermann'
 require 'zk'
 require 'json'
+require 'hermann/errors'
 
 module Hermann
   module Discovery
-    class NoBrokersError < StandardError; end
+
 
     # Communicates with Zookeeper to discover kafka broker ids
     #
@@ -24,12 +25,16 @@ module Hermann
       #    of 20 times the tickTime2 times the tick time set on server"
       #
       # @return [String] comma separated list of brokers
+      #
+      # @raises [NoBrokersError] if could not discover brokers thru zookeeper
       def get_brokers(timeout=0)
         brokers = []
         ZK.open(zookeepers, {:timeout => timeout}) do |zk|
           brokers = fetch_brokers(zk)
         end
-        raise NoBrokersError if brokers.empty?
+        if brokers.empty?
+          raise Hermann::Errors::NoBrokersError
+        end
         brokers.join(',')
       end
 

data/lib/hermann/errors.rb

@@ -3,6 +3,12 @@ module Hermann
   module Errors
     # Error for connectivity problems with the Kafka brokers
     class ConnectivityError; end;
+
+    # For passing incorrect config and options to kafka
+    class ConfigurationError < StandardError; end
+
+    # cannot discover brokers from zookeeper
+    class NoBrokersError < StandardError; end
   end
 end
 

data/lib/hermann/producer.rb

@@ -12,13 +12,17 @@ module Hermann
   class Producer
     attr_reader :topic, :brokers, :internal, :children
 
-    def initialize(topic, brokers)
+    # Initialize a producer object with a default topic and broker list
+    #
+    # @param [String] topic The default topic to use for pushing messages
+    # @param [Array] brokers An array of "host:port" strings for the brokers
+    def initialize(topic, brokers, opts={})
       @topic = topic
       @brokers = brokers
       if RUBY_PLATFORM == "java"
-        @internal = Hermann::Provider::JavaProducer.new(topic, brokers)
+        @internal = Hermann::Provider::JavaProducer.new(brokers, opts)
       else
-        @internal = Hermann::Lib::Producer.new(topic, brokers)
+        @internal = Hermann::Lib::Producer.new(brokers)
       end
       # We're tracking children so we can make sure that at Producer exit we
       # make a reasonable attempt to clean up outstanding result objects
@@ -42,23 +46,31 @@ module Hermann
 
     # Push a value onto the Kafka topic passed to this +Producer+
     #
-    # @param [Array] value An array of values to push, will push each one
-    #   separately
     # @param [Object] value A single object to push
+    # @param [Hash] opts to pass to push method
+    # @option opts [String] :topic The topic to push messages to
+    #
     # @return [Hermann::Result] A future-like object which will store the
     #   result from the broker
-    def push(value)
-      result = create_result
+    def push(value, opts={})
+      topic = opts[:topic] || @topic
+      result = nil
 
       if value.kind_of? Array
-        return value.map { |e| self.push(e) }
-      else
-        if RUBY_PLATFORM == "java"
-          result = @internal.push_single(value)
+        return value.map { |e| self.push(e, opts) }
+      end
+
+      if RUBY_PLATFORM == "java"
+        result = @internal.push_single(value, topic, nil)
+        unless result.nil?
           @children << result
-        else
-          @internal.push_single(value, result)
         end
+        # Reaping children on the push just to make sure that it does get
+        # called correctly and we don't leak memory
+        reap_children
+      else
+        result = create_result
+        @internal.push_single(value, topic, result)
       end
 
       return result
@@ -118,7 +130,7 @@ module Hermann
     end
 
     # Perform the actual reactor tick
-    # @raises [StandardError[ in case of underlying failures in librdkafka
+    # @raises [StandardError] in case of underlying failures in librdkafka
     def execute_tick(timeout)
       if timeout == 0
         @internal.tick(0)

data/lib/hermann/provider/java_producer.rb

@@ -4,61 +4,103 @@ require 'json'
 
 module Hermann
   module Provider
-
     # This class simulates the kafka producer class within a java environment.
     # If the producer throw an exception within the Promise a call to +.value!+
     # will raise the exception and the rejected flag will be set to true
     #
     class JavaProducer
-      attr_accessor :topic, :producer
+      attr_accessor :producer
+
 
-      def initialize(topic, brokers)
-        @topic      = topic
-        properties  = create_properties(:brokers => brokers)
+      # Instantiate JavaProducer
+      #
+      # @params [String] list of brokers
+      #
+      # @params [Hash] hash of kafka attributes, overrides defaults
+      #
+      # @raises [RuntimeError] if brokers string is nil/empty
+      #
+      # ==== Examples
+      #
+      # JavaProducer.new('0:9092', {'request.required.acks' => '1'})
+      #
+      def initialize(brokers, opts={})
+        properties  = create_properties(brokers, opts)
         config      = create_config(properties)
         @producer   = JavaApiUtil::Producer.new(config)
       end
 
       DEFAULTS = {
-                    :string_encoder => 'kafka.serializer.StringEncoder',
-                    :partitioner    => 'kafka.producer.DefaultPartitioner',
-                    :required_acks  => "1"
+                    'serializer.class'      => 'kafka.serializer.StringEncoder',
+                    'partitioner.class'     => 'kafka.producer.DefaultPartitioner',
+                    'request.required.acks' => '1'
                   }.freeze
 
       # Push a value onto the Kafka topic passed to this +Producer+
       #
       # @param [Object] value A single object to push
+      # @param [String] topic to push message to
       #
       # @return +Concurrent::Promise+ Representa a promise to send the
       #   data to the kafka broker.  Upon execution the Promise's status
       #   will be set
-      def push_single(msg)
+      def push_single(msg, topic, unused)
         Concurrent::Promise.execute {
-          data = ProducerUtil::KeyedMessage.new(@topic, msg)
+          data = ProducerUtil::KeyedMessage.new(topic, msg)
           @producer.send(data)
         }
       end
 
+      # No-op for now
+      def connected?
+        return false
+      end
+
+      # No-op for now
+      def errored?
+        return false
+      end
+
+      # No-op for now
+      def connect(timeout=0)
+        nil
+      end
+
       private
+
+        # Creates a ProducerConfig object
+        #
+        # @param [Properties] object with broker properties
+        #
         # @return [ProducerConfig] - packaged config for +Producer+
         def create_config(properties)
           ProducerUtil::ProducerConfig.new(properties)
         end
 
+        # Creates Properties Object
+        #
+        # @param [Hash] brokers passed into this function
+        # @option args [String] :brokers - string of brokers
+        #
         # @return [Properties] properties object for creating +ProducerConfig+
-        def create_properties(args={})
-          brokers     = args[:brokers]
-          str_encoder = DEFAULTS[:string_encoder]
-          partitioner = DEFAULTS[:partitioner]
-          acks        = DEFAULTS[:required_acks]
-
+        #
+        # @raises [RuntimeError] if options does not contain key value strings
+        def create_properties(brokers, opts={})
+          brokers = { 'metadata.broker.list' => brokers }
+          options = DEFAULTS.merge(brokers).merge(opts)
           properties = JavaUtil::Properties.new
-          properties.put('metadata.broker.list',  brokers)
-          properties.put('serializer.class',      str_encoder)
-          properties.put('partitioner.class',     partitioner)
-          properties.put('request.required.acks', acks)
+          options.each do |key, val|
+            validate_property!(key, val)
+            properties.put(key, val)
+          end
           properties
         end
+
+        def validate_property!(key, val)
+          if key.to_s.empty? || val.to_s.empty?
+            raise Hermann::Errors::ConfigurationError, "Invalid Broker Properties"
+          end
+        end
     end
   end
 end

data/lib/hermann/version.rb

@@ -1,3 +1,3 @@
 module Hermann
-  VERSION = '0.17.0'
+  VERSION = '0.18.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: hermann
 version: !ruby/object:Gem::Version 
-  version: 0.17.0
+  version: 0.18.0
 platform: ruby
 authors: 
 - R. Tyler Croy
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-10-10 00:00:00 Z
+date: 2014-10-14 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: concurrent-ruby