hermann 0.16.0 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 --- 
 SHA1: 
-  metadata.gz: 8194f73012572376c8cd14ae68042da11fc79dfc
-  data.tar.gz: 355dacd9317c634b430e4a7f32216aaccf07a308
+  metadata.gz: b08abedfd6c15b1c7c1aa5dfe66315501c212512
+  data.tar.gz: 3bf7306d482a260fd73004e4d8087cd5f3a1f063
 SHA512: 
-  metadata.gz: a1d4dd0e592433519e086d889b4362ff1e69f805f121517812bffd4b67ea1a77fb1d941fe0e017968ad791fe23506371831070a905259600f7154f8adca2cd7b
-  data.tar.gz: db0b9f012f34e31c23268b4a93cbf40ea38a2b8239c3fe4fb7ca140d339259adffec0aa31f6098466ed57e107149c537ed43630694c6cea852c4048512266db9
+  metadata.gz: 901a4eecf276229c561186f5be901b6ed87313d35d6878365621ee0302328870f24fe02661620d633f4e55462aaf953a8036c3da88d5d6e6d66c7e71e9e6a9d4
+  data.tar.gz: 300db74c52df5be4f01078b6098c9a0cd528da811c4681e2fb0dde337c7b5b9bfe1cbe645769f9c616adbc9cfe5a841d9413f2d51208518e2dd1a20a4fce57cf

data/Rakefile

@@ -12,7 +12,15 @@ Rake::ExtensionTask.new do |t|
 end
 
 RSpec::Core::RakeTask.new(:spec) do |r|
-  r.rspec_opts = '--tag ~type:integration'
+  options = ['--tag ~type:integration']
+
+  if RUBY_PLATFORM == 'java'
+    options << '--tag ~platform:mri'
+  else
+    options << '--tag ~platform:java'
+  end
+
+  r.rspec_opts = options.join(' ')
 end
 
 namespace :spec do
@@ -29,5 +37,10 @@ end
 
 task :build => [:compile]
 task :clean => [:removetmp]
-task :default => [:clean, :build, :spec]
+
+if RUBY_PLATFORM == 'java'
+  task :default => [:clean, :spec]
+else
+  task :default => [:clean, :build, :spec]
+end
 

data/lib/hermann/consumer.rb

@@ -1,5 +1,8 @@
 require 'hermann'
-require 'hermann_lib'
+
+unless Hermann.jruby?
+  require 'hermann_lib'
+end
 
 module Hermann
   class Consumer
@@ -9,7 +12,9 @@ module Hermann
       @topic = topic
       @brokers = brokers
       @partition = partition
-      @internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
+      unless Hermann.jruby?
+        @internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
+      end
     end
 
     def consume(&block)

data/lib/hermann/discovery/zookeeper.rb

@@ -0,0 +1,80 @@
+require 'hermann'
+require 'zk'
+require 'json'
+
+module Hermann
+  module Discovery
+    class NoBrokersError < StandardError; end
+
+    # Communicates with Zookeeper to discover kafka broker ids
+    #
+    class Zookeeper
+      attr_reader :zookeepers
+
+      BROKERS_PATH = "/brokers/ids".freeze
+
+      def initialize(zookeepers)
+        @zookeepers = zookeepers
+      end
+
+      # Gets comma separated string of brokers
+      #
+      # @param [Fixnum] timeout to connect to zookeeper, "2 times the
+      #    tickTime (as set in the server configuration) and a maximum
+      #    of 20 times the tickTime2 times the tick time set on server"
+      #
+      # @return [String] comma separated list of brokers
+      def get_brokers(timeout=0)
+        brokers = []
+        ZK.open(zookeepers, {:timeout => timeout}) do |zk|
+          brokers = fetch_brokers(zk)
+        end
+        raise NoBrokersError if brokers.empty?
+        brokers.join(',')
+      end
+
+      private
+
+        # Gets an Array of broker strings
+        #
+        # @param [ZK::Client] zookeeper client
+        #
+        # @return array of broker strings
+        def fetch_brokers(zk)
+          brokers = []
+          zk.children(BROKERS_PATH).each do |id|
+            node = fetch_znode(zk, id)
+            next if node.nil? # whatever error could happen from ZK#get
+            brokers << format_broker_from_znode(node)
+          end
+          brokers.compact
+        end
+
+        # Gets node from zookeeper
+        #
+        # @param [ZK::Client] zookeeper client
+        # @param [Fixnum] kafka broker
+        #
+        # @return [String] node data
+        def fetch_znode(zk, id)
+          zk.get("#{BROKERS_PATH}/#{id}")[0]
+        rescue ZK::Exceptions::NoNode
+          nil
+        end
+
+        # Formats the node data into string
+        #
+        # @param [String] node data
+        #
+        # @return [String] formatted node data or empty string if error
+        def format_broker_from_znode(znode)
+          hash = JSON.parse(znode)
+          host = hash['host']
+          port = hash['port']
+          host && port ? "#{host}:#{port}" : nil
+        rescue JSON::ParserError
+          nil
+        end
+    end
+  end
+end

data/lib/hermann/producer.rb

@@ -1,6 +1,12 @@
 require 'hermann'
 require 'hermann/result'
-require 'hermann_lib'
+
+
+if RUBY_PLATFORM == "java"
+  require 'hermann/provider/java_producer'
+else
+  require 'hermann_lib'
+end
 
 module Hermann
   class Producer
@@ -9,7 +15,11 @@ module Hermann
     def initialize(topic, brokers)
       @topic = topic
       @brokers = brokers
-      @internal = Hermann::Lib::Producer.new(topic, brokers)
+      if RUBY_PLATFORM == "java"
+        @internal = Hermann::Provider::JavaProducer.new(topic, brokers)
+      else
+        @internal = Hermann::Lib::Producer.new(topic, brokers)
+      end
       # We're tracking children so we can make sure that at Producer exit we
       # make a reasonable attempt to clean up outstanding result objects
       @children = []
@@ -43,7 +53,12 @@ module Hermann
       if value.kind_of? Array
         return value.map { |e| self.push(e) }
       else
-        @internal.push_single(value, result)
+        if RUBY_PLATFORM == "java"
+          result = @internal.push_single(value)
+          @children << result
+        else
+          @internal.push_single(value, result)
+        end
       end
 
       return result

data/lib/hermann/provider/java_producer.rb

@@ -0,0 +1,64 @@
+require 'hermann'
+require 'concurrent'
+require 'json'
+
+module Hermann
+  module Provider
+
+    # This class simulates the kafka producer class within a java environment.
+    # If the producer throw an exception within the Promise a call to +.value!+
+    # will raise the exception and the rejected flag will be set to true
+    #
+    class JavaProducer
+      attr_accessor :topic, :producer
+
+      def initialize(topic, brokers)
+        @topic      = topic
+        properties  = create_properties(:brokers => brokers)
+        config      = create_config(properties)
+        @producer   = JavaApiUtil::Producer.new(config)
+      end
+
+      DEFAULTS = {
+                    :string_encoder => 'kafka.serializer.StringEncoder',
+                    :partitioner    => 'kafka.producer.DefaultPartitioner',
+                    :required_acks  => "1"
+                  }.freeze
+
+      # Push a value onto the Kafka topic passed to this +Producer+
+      #
+      # @param [Object] value A single object to push
+      #
+      # @return +Concurrent::Promise+ Representa a promise to send the
+      #   data to the kafka broker.  Upon execution the Promise's status
+      #   will be set
+      def push_single(msg)
+        Concurrent::Promise.execute {
+          data = ProducerUtil::KeyedMessage.new(@topic, msg)
+          @producer.send(data)
+        }
+      end
+
+      private
+        # @return [ProducerConfig] - packaged config for +Producer+
+        def create_config(properties)
+          ProducerUtil::ProducerConfig.new(properties)
+        end
+
+        # @return [Properties] properties object for creating +ProducerConfig+
+        def create_properties(args={})
+          brokers     = args[:brokers]
+          str_encoder = DEFAULTS[:string_encoder]
+          partitioner = DEFAULTS[:partitioner]
+          acks        = DEFAULTS[:required_acks]
+
+          properties = JavaUtil::Properties.new
+          properties.put('metadata.broker.list',  brokers)
+          properties.put('serializer.class',      str_encoder)
+          properties.put('partitioner.class',     partitioner)
+          properties.put('request.required.acks', acks)
+          properties
+        end
+    end
+  end
+end

data/lib/hermann/version.rb

@@ -1,3 +1,3 @@
 module Hermann
-  VERSION = '0.16.0'
+  VERSION = '0.17.0'
 end

data/lib/hermann.rb

@@ -1,2 +1,20 @@
 module Hermann
+  def self.jruby?
+    return RUBY_PLATFORM == "java"
+  end
+
+  if self.jruby?
+    require 'java'
+    require 'hermann_jars'
+
+    module JavaUtil
+      include_package 'java.util'
+    end
+    module ProducerUtil
+      include_package 'kafka.producer'
+    end
+    module JavaApiUtil
+      include_package 'kafka.javaapi.producer'
+    end
+  end
 end

data/lib/hermann_jars.rb

@@ -0,0 +1,15 @@
+# this is a generated file, to avoid over-writing it just delete this comment
+require 'jar_dependencies'
+
+require_jar( 'org.slf4j', 'slf4j-api', '1.7.2' )
+require_jar( 'org.scala-lang', 'scala-library', '2.10.1' )
+require_jar( 'log4j', 'log4j', '1.2.14' )
+require_jar( 'com.yammer.metrics', 'metrics-core', '2.2.0' )
+require_jar( 'org.apache.zookeeper', 'zookeeper', '3.3.4' )
+require_jar( 'net.sf.jopt-simple', 'jopt-simple', '3.2' )
+require_jar( 'org.apache.kafka', 'kafka_2.10', '0.8.1.1' )
+require_jar( 'jline', 'jline', '0.9.94' )
+require_jar( 'com.101tec', 'zkclient', '0.3' )
+require_jar( 'org.mod4j.org.eclipse.xtext', 'log4j', '1.2.15' )
+require_jar( 'junit', 'junit', '3.8.1' )
+require_jar( 'org.xerial.snappy', 'snappy-java', '1.0.5' )

metadata

@@ -1,31 +1,51 @@
 --- !ruby/object:Gem::Specification 
 name: hermann
 version: !ruby/object:Gem::Version 
-  version: 0.16.0
+  version: 0.17.0
 platform: ruby
 authors: 
-- Stan Campbell
 - R. Tyler Croy
+- Stan Campbell
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2014-09-12 00:00:00 Z
+date: 2014-10-10 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: mini_portile
+  name: concurrent-ruby
   requirement: &id001 !ruby/object:Gem::Requirement 
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 0.6.0
+        version: 0.7.0
   type: :runtime
   prerelease: false
   version_requirements: *id001
-description: Ruby gem wrapper for librdkafka
+- !ruby/object:Gem::Dependency 
+  name: zk
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.9.4
+  type: :runtime
+  prerelease: false
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: mini_portile
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *id003
+description: Ruby gem for talking to Kafka
 email: 
-- stan.campbell3@gmail.com
 - rtyler.croy@lookout.com
+- stan.campbell3@gmail.com
 executables: []
 
 extensions: 
@@ -40,11 +60,14 @@ files:
 - ext/patches/librdkafka/0006-Update-some-headers-to-include-the-right-headers-to-.patch
 - lib/hermann.rb
 - lib/hermann/consumer.rb
+- lib/hermann/discovery/zookeeper.rb
 - lib/hermann/errors.rb
 - lib/hermann/producer.rb
+- lib/hermann/provider/java_producer.rb
 - lib/hermann/result.rb
 - lib/hermann/timeout.rb
 - lib/hermann/version.rb
+- lib/hermann_jars.rb
 homepage: https://github.com/lookout/Hermann
 licenses: 
 - MIT
@@ -58,19 +81,19 @@ require_paths:
 - ext/hermann
 required_ruby_version: !ruby/object:Gem::Requirement 
   requirements: 
-  - &id002 
+  - &id004 
     - ">="
     - !ruby/object:Gem::Version 
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
-  - *id002
+  - *id004
 requirements: []
 
 rubyforge_project: 
 rubygems_version: 2.4.1
 signing_key: 
 specification_version: 3
-summary: A Kafka consumer/producer gem based on the librdkafka C library.
+summary: A Kafka consumer/producer gem supporting both MRI and JRuby
 test_files: []