hermann 0.18.1-java

data/ext/hermann/hermann_lib.h

@@ -0,0 +1,107 @@
+/*
+ * hermann_lib.h - Ruby wrapper for the librdkafka library
+ *
+ * Copyright (c) 2014 Stan Campbell
+ * Copyright (c) 2014 Lookout, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *	this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *	this list of conditions and the following disclaimer in the documentation
+ *	and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HERMANN_H
+#define HERMANN_H
+
+#include <ruby.h>
+
+#include <ctype.h>
+#include <signal.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <syslog.h>
+#include <sys/time.h>
+#include <errno.h>
+
+#include <librdkafka/rdkafka.h>
+
+#ifdef TRACE
+#define TRACER(...) do {  \
+	fprintf(stderr, "%i:%s()> ", __LINE__, __PRETTY_FUNCTION__); \
+	fprintf(stderr, __VA_ARGS__); \
+	fflush(stderr); \
+					} while (0)
+#else
+#define TRACER(...) do { } while (0)
+#endif
+
+// Holds the defined Ruby module for Hermann
+static VALUE hermann_module;
+
+#define HERMANN_MAX_ERRSTR_LEN 512
+
+static int DEBUG = 0;
+
+// Should we expect rb_thread_blocking_region to be present?
+// #define RB_THREAD_BLOCKING_REGION
+#undef RB_THREAD_BLOCKING_REGION
+
+static 	enum {
+	OUTPUT_HEXDUMP,
+	OUTPUT_RAW,
+} output = OUTPUT_HEXDUMP;
+
+typedef struct HermannInstanceConfig {
+	char *topic;
+
+	/* Kafka configuration */
+	rd_kafka_t *rk;
+	rd_kafka_topic_t *rkt;
+	char *brokers;
+	int partition;
+	rd_kafka_topic_conf_t *topic_conf;
+	char errstr[512];
+	rd_kafka_conf_t *conf;
+	const char *debug;
+	int64_t start_offset;
+	int do_conf_dump;
+
+	int run;
+	int exit_eof;
+	int quiet;
+
+	int isInitialized;
+	int isConnected;
+
+	int isErrored;
+	char *error;
+} HermannInstanceConfig;
+
+typedef HermannInstanceConfig hermann_conf_t;
+
+typedef struct {
+	/* Hermann::Lib::Producer */
+	hermann_conf_t *producer;
+	/* Hermann::Result */
+	VALUE result;
+} hermann_push_ctx_t;
+
+#endif

data/ext/patches/librdkafka/0006-Update-some-headers-to-include-the-right-headers-to-.patch

@@ -0,0 +1,57 @@
+From 888ca33b571d99e877d665235b822f7c961c8fdb Mon Sep 17 00:00:00 2001
+From: "R. Tyler Croy" <tyler@monkeypox.org>
+Date: Thu, 28 Aug 2014 16:24:04 -0700
+Subject: [PATCH 6/8] Update some headers to include the right headers to build
+ on FreeBSD
+
+---
+ src/rd.h     | 9 +++++++++
+ src/rdaddr.h | 4 ++++
+ 2 files changed, 13 insertions(+)
+
+diff --git a/src/rd.h b/src/rd.h
+index c31501e..4789493 100644
+--- a/src/rd.h
++++ b/src/rd.h
+@@ -37,7 +37,11 @@
+ #include <errno.h>
+ #include <time.h>
+ #include <sys/time.h>
++
++#ifndef __FreeBSD__
++/* alloca(3) is in stdlib on FreeBSD */
+ #include <alloca.h>
++#endif
+ #include <assert.h>
+ #include <pthread.h>
+ 
+@@ -110,6 +114,11 @@
+ # endif
+ #endif /* sun */
+ 
++#ifdef __FreeBSD__
++/* FreeBSD defines be64toh() in sys/endian.h */
++#include <sys/endian.h>
++#endif
++
+ #ifndef be64toh
+ #ifndef __APPLE__
+ #ifndef sun
+diff --git a/src/rdaddr.h b/src/rdaddr.h
+index 0b37354..e55bd55 100644
+--- a/src/rdaddr.h
++++ b/src/rdaddr.h
+@@ -32,6 +32,10 @@
+ #include <arpa/inet.h>
+ #include <netdb.h>
+ 
++#ifdef __FreeBSD__
++#include <sys/socket.h>
++#endif
++
+ /**
+  * rd_sockaddr_inx_t is a union for either ipv4 or ipv6 sockaddrs.
+  * It provides conveniant abstraction of AF_INET* agnostic operations.
+-- 
+1.9.0
+

data/lib/hermann/consumer.rb

@@ -0,0 +1,24 @@
+require 'hermann'
+
+unless Hermann.jruby?
+  require 'hermann_lib'
+end
+
+module Hermann
+  class Consumer
+    attr_reader :topic, :brokers, :partition, :internal
+
+    def initialize(topic, brokers, partition)
+      @topic = topic
+      @brokers = brokers
+      @partition = partition
+      unless Hermann.jruby?
+        @internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
+      end
+    end
+
+    def consume(&block)
+      @internal.consume(&block)
+    end
+  end
+end

data/lib/hermann/discovery/zookeeper.rb

@@ -0,0 +1,85 @@
+require 'hermann'
+require 'zk'
+require 'json'
+require 'hermann/errors'
+
+module Hermann
+  module Discovery
+
+
+    # Communicates with Zookeeper to discover kafka broker ids
+    #
+    class Zookeeper
+      attr_reader :zookeepers
+
+      BROKERS_PATH = "/brokers/ids".freeze
+
+      def initialize(zookeepers)
+        @zookeepers = zookeepers
+      end
+
+      # Gets comma separated string of brokers
+      #
+      # @param [Fixnum] timeout to connect to zookeeper, "2 times the
+      #    tickTime (as set in the server configuration) and a maximum
+      #    of 20 times the tickTime2 times the tick time set on server"
+      #
+      # @return [String] comma separated list of brokers
+      #
+      # @raises [NoBrokersError] if could not discover brokers thru zookeeper
+      def get_brokers(timeout=0)
+        brokers = []
+        ZK.open(zookeepers, {:timeout => timeout}) do |zk|
+          brokers = fetch_brokers(zk)
+        end
+        if brokers.empty?
+          raise Hermann::Errors::NoBrokersError
+        end
+        brokers.join(',')
+      end
+
+      private
+
+        # Gets an Array of broker strings
+        #
+        # @param [ZK::Client] zookeeper client
+        #
+        # @return array of broker strings
+        def fetch_brokers(zk)
+          brokers = []
+          zk.children(BROKERS_PATH).each do |id|
+            node = fetch_znode(zk, id)
+            next if node.nil? # whatever error could happen from ZK#get
+            brokers << format_broker_from_znode(node)
+          end
+          brokers.compact
+        end
+
+        # Gets node from zookeeper
+        #
+        # @param [ZK::Client] zookeeper client
+        # @param [Fixnum] kafka broker
+        #
+        # @return [String] node data
+        def fetch_znode(zk, id)
+          zk.get("#{BROKERS_PATH}/#{id}")[0]
+        rescue ZK::Exceptions::NoNode
+          nil
+        end
+
+        # Formats the node data into string
+        #
+        # @param [String] node data
+        #
+        # @return [String] formatted node data or empty string if error
+        def format_broker_from_znode(znode)
+          hash = JSON.parse(znode)
+          host = hash['host']
+          port = hash['port']
+          host && port ? "#{host}:#{port}" : nil
+        rescue JSON::ParserError
+          nil
+        end
+    end
+  end
+end

data/lib/hermann/errors.rb

@@ -0,0 +1,14 @@
+
+module Hermann
+  module Errors
+    # Error for connectivity problems with the Kafka brokers
+    class ConnectivityError; end;
+
+    # For passing incorrect config and options to kafka
+    class ConfigurationError < StandardError; end
+
+    # cannot discover brokers from zookeeper
+    class NoBrokersError < StandardError; end
+  end
+end
+

data/lib/hermann/producer.rb

@@ -0,0 +1,150 @@
+require 'hermann'
+require 'hermann/result'
+
+
+if RUBY_PLATFORM == "java"
+  require 'hermann/provider/java_producer'
+else
+  require 'hermann_lib'
+end
+
+module Hermann
+  class Producer
+    attr_reader :topic, :brokers, :internal, :children
+
+    # Initialize a producer object with a default topic and broker list
+    #
+    # @param [String] topic The default topic to use for pushing messages
+    # @param [Array] brokers An array of "host:port" strings for the brokers
+    def initialize(topic, brokers, opts={})
+      @topic = topic
+      @brokers = brokers
+      if RUBY_PLATFORM == "java"
+        @internal = Hermann::Provider::JavaProducer.new(brokers, opts)
+      else
+        @internal = Hermann::Lib::Producer.new(brokers)
+      end
+      # We're tracking children so we can make sure that at Producer exit we
+      # make a reasonable attempt to clean up outstanding result objects
+      @children = []
+    end
+
+    # @return [Boolean] True if our underlying producer object thinks it's
+    #   connected to a Kafka broker
+    def connected?
+      return @internal.connected?
+    end
+
+    # @return [Boolean] True if the underlying producer object has errored
+    def errored?
+      return @internal.errored?
+    end
+
+    def connect(timeout=0)
+      return @internal.connect(timeout * 1000)
+    end
+
+    # Push a value onto the Kafka topic passed to this +Producer+
+    #
+    # @param [Object] value A single object to push
+    # @param [Hash] opts to pass to push method
+    # @option opts [String] :topic The topic to push messages to
+    #
+    # @return [Hermann::Result] A future-like object which will store the
+    #   result from the broker
+    def push(value, opts={})
+      topic = opts[:topic] || @topic
+      result = nil
+
+      if value.kind_of? Array
+        return value.map { |e| self.push(e, opts) }
+      end
+
+      if RUBY_PLATFORM == "java"
+        result = @internal.push_single(value, topic, nil)
+        unless result.nil?
+          @children << result
+        end
+        # Reaping children on the push just to make sure that it does get
+        # called correctly and we don't leak memory
+        reap_children
+      else
+        result = create_result
+        @internal.push_single(value, topic, result)
+      end
+
+      return result
+    end
+
+    # Create a +Hermann::Result+ that is tracked in the Producer's children
+    # array
+    #
+    # @return [Hermann::Result] A new, unused, result
+    def create_result
+      @children << Hermann::Result.new(self)
+      return @children.last
+    end
+
+    # Tick the underlying librdkafka reacter and clean up any unreaped but
+    # reapable children results
+    #
+    # @param [FixNum] timeout Seconds to block on the internal reactor
+    # @return [FixNum] Number of +Hermann::Result+ children reaped
+    def tick_reactor(timeout=0)
+      begin
+        execute_tick(rounded_timeout(timeout))
+      rescue StandardError => ex
+        @children.each do |child|
+          # Skip over any children that should already be reaped for other
+          # reasons
+          next if child.completed?
+          # Propagate errors to the remaining children
+          child.internal_set_error(ex)
+        end
+      end
+
+      # Reaping the children at this point will also reap any children marked
+      # as errored by an exception out of #execute_tick
+      return reap_children
+    end
+
+    # @return [FixNum] number of children reaped
+    def reap_children
+      # Filter all children who are no longer pending/fulfilled
+      total_children = @children.size
+
+      @children = @children.reject { |c| c.completed? }
+
+      return (total_children - children.size)
+    end
+
+
+    private
+
+    def rounded_timeout(timeout)
+      # Handle negative numbers, those can be zero
+      return 0 if (timeout < 0)
+      # Since we're going to sleep for each second, round any potential floats
+      # off
+      return timeout.round if timeout.kind_of?(Float)
+      return timeout
+    end
+
+    # Perform the actual reactor tick
+    # @raises [StandardError] in case of underlying failures in librdkafka
+    def execute_tick(timeout)
+      if timeout == 0
+        @internal.tick(0)
+      else
+        (timeout * 2).times do
+          # We're going to Thread#sleep in Ruby to avoid a
+          # pthread_cond_timedwait(3) inside of librdkafka
+          events = @internal.tick(0)
+          # If we find events, break out early
+          break if events > 0
+          sleep 0.5
+        end
+      end
+    end
+  end
+end

data/lib/hermann/provider/java_producer.rb

@@ -0,0 +1,106 @@
+require 'hermann'
+require 'concurrent'
+require 'json'
+
+module Hermann
+  module Provider
+    # This class simulates the kafka producer class within a java environment.
+    # If the producer throw an exception within the Promise a call to +.value!+
+    # will raise the exception and the rejected flag will be set to true
+    #
+    class JavaProducer
+      attr_accessor :producer
+
+
+      # Instantiate JavaProducer
+      #
+      # @params [String] list of brokers
+      #
+      # @params [Hash] hash of kafka attributes, overrides defaults
+      #
+      # @raises [RuntimeError] if brokers string is nil/empty
+      #
+      # ==== Examples
+      #
+      # JavaProducer.new('0:9092', {'request.required.acks' => '1'})
+      #
+      def initialize(brokers, opts={})
+        properties  = create_properties(brokers, opts)
+        config      = create_config(properties)
+        @producer   = JavaApiUtil::Producer.new(config)
+      end
+
+      DEFAULTS = {
+                    'serializer.class'      => 'kafka.serializer.StringEncoder',
+                    'partitioner.class'     => 'kafka.producer.DefaultPartitioner',
+                    'request.required.acks' => '1'
+                  }.freeze
+
+      # Push a value onto the Kafka topic passed to this +Producer+
+      #
+      # @param [Object] value A single object to push
+      # @param [String] topic to push message to
+      #
+      # @return +Concurrent::Promise+ Representa a promise to send the
+      #   data to the kafka broker.  Upon execution the Promise's status
+      #   will be set
+      def push_single(msg, topic, unused)
+        Concurrent::Promise.execute {
+          data = ProducerUtil::KeyedMessage.new(topic, msg)
+          @producer.send(data)
+        }
+      end
+
+      # No-op for now
+      def connected?
+        return false
+      end
+
+      # No-op for now
+      def errored?
+        return false
+      end
+
+      # No-op for now
+      def connect(timeout=0)
+        nil
+      end
+
+      private
+
+        # Creates a ProducerConfig object
+        #
+        # @param [Properties] object with broker properties
+        #
+        # @return [ProducerConfig] - packaged config for +Producer+
+        def create_config(properties)
+          ProducerUtil::ProducerConfig.new(properties)
+        end
+
+        # Creates Properties Object
+        #
+        # @param [Hash] brokers passed into this function
+        # @option args [String] :brokers - string of brokers
+        #
+        # @return [Properties] properties object for creating +ProducerConfig+
+        #
+        # @raises [RuntimeError] if options does not contain key value strings
+        def create_properties(brokers, opts={})
+          brokers = { 'metadata.broker.list' => brokers }
+          options = DEFAULTS.merge(brokers).merge(opts)
+          properties = JavaUtil::Properties.new
+          options.each do |key, val|
+            validate_property!(key, val)
+            properties.put(key, val)
+          end
+          properties
+        end
+
+        def validate_property!(key, val)
+          if key.to_s.empty? || val.to_s.empty?
+            raise Hermann::Errors::ConfigurationError, "Invalid Broker Properties"
+          end
+        end
+    end
+  end
+end

data/lib/hermann/result.rb

@@ -0,0 +1,74 @@
+
+module Hermann
+  class Result
+    attr_reader :reason, :state
+
+    STATES = [:pending,
+              :rejected,
+              :fulfilled,
+              :unfulfilled,
+              ].freeze
+
+    def initialize(producer)
+      @producer = producer
+      @reason = nil
+      @value = nil
+      @state = :unfulfilled
+    end
+
+    STATES.each do |state|
+      define_method("#{state}?".to_sym) do
+        return @state == state
+      end
+    end
+
+    # @return [Boolean] True if this child can be reaped
+    def completed?
+      return true if rejected? || fulfilled?
+      return false
+    end
+
+    # Access the value of the future
+    #
+    # @param [FixNum] timeout Seconds to wait on the underlying machinery for a
+    #   result
+    # @return [NilClass] nil if no value could be received in the time alotted
+    # @return [Object]
+    def value(timeout=0)
+      @producer.tick_reactor(timeout)
+      return @value
+    end
+
+    # INTERNAL METHOD ONLY. Do not use
+    #
+    # This method will be invoked by the underlying extension to indicate set
+    # the actual value after a callback has completed
+    #
+    # @param [Object] value The actual resulting value
+    # @param [Boolean] is_error True if the result was errored for whatever
+    #   reason
+    def internal_set_value(value, is_error)
+      @value = value
+
+      if is_error
+        puts "Hermann::Result#set_internal_value(#{value.class}:\"#{value}\", error?:#{is_error})"
+        @state = :rejected
+      else
+        @state = :fulfilled
+      end
+    end
+
+    # INTERNAL METHOD ONLY. Do not use
+    #
+    # This method will set our internal #reason with the details from the
+    # exception
+    #
+    # @param [Exception] exception
+    def internal_set_error(exception)
+      return if exception.nil?
+
+      @reason = exception
+      @state = :rejected
+    end
+  end
+end

data/lib/hermann/timeout.rb

@@ -0,0 +1,37 @@
+begin
+  require 'system_timer'
+  module Hermann
+    class Timeout
+      USE_SYSTEM_TIMER = true
+    end
+  end
+rescue LoadError
+  require 'timeout'
+
+  if RUBY_VERSION == '1.8.7'
+    puts ">>> You are running on 1.8.7 without SystemTimer"
+    puts ">>> which means Hermann::Timeout will not work as expected"
+  end
+  module Hermann
+    class Timeout
+      USE_SYSTEM_TIMER = false
+    end
+  end
+end
+
+module Hermann
+  class Timeout
+    def self.system_timer?
+      Hermann::Timeout::USE_SYSTEM_TIMER
+    end
+
+    def self.timeout(seconds, klass=nil, &block)
+      if system_timer?
+        SystemTimer.timeout_after(seconds, klass, &block)
+      else
+        ::Timeout.timeout(seconds, klass, &block)
+      end
+    end
+  end
+end
+

data/lib/hermann/version.rb

@@ -0,0 +1,3 @@
+module Hermann
+  VERSION = '0.18.1'
+end

data/lib/hermann.rb

@@ -0,0 +1,20 @@
+module Hermann
+  def self.jruby?
+    return RUBY_PLATFORM == "java"
+  end
+
+  if self.jruby?
+    require 'java'
+    require 'hermann_jars'
+
+    module JavaUtil
+      include_package 'java.util'
+    end
+    module ProducerUtil
+      include_package 'kafka.producer'
+    end
+    module JavaApiUtil
+      include_package 'kafka.javaapi.producer'
+    end
+  end
+end

data/lib/hermann_jars.rb

@@ -0,0 +1,15 @@
+# this is a generated file, to avoid over-writing it just delete this comment
+require 'jar_dependencies'
+
+require_jar( 'org.slf4j', 'slf4j-api', '1.7.2' )
+require_jar( 'org.scala-lang', 'scala-library', '2.10.1' )
+require_jar( 'log4j', 'log4j', '1.2.14' )
+require_jar( 'com.yammer.metrics', 'metrics-core', '2.2.0' )
+require_jar( 'org.apache.zookeeper', 'zookeeper', '3.3.4' )
+require_jar( 'net.sf.jopt-simple', 'jopt-simple', '3.2' )
+require_jar( 'org.apache.kafka', 'kafka_2.10', '0.8.1.1' )
+require_jar( 'jline', 'jline', '0.9.94' )
+require_jar( 'com.101tec', 'zkclient', '0.3' )
+require_jar( 'org.mod4j.org.eclipse.xtext', 'log4j', '1.2.15' )
+require_jar( 'junit', 'junit', '3.8.1' )
+require_jar( 'org.xerial.snappy', 'snappy-java', '1.0.5' )