hermann 0.16.0

data/ext/hermann/hermann_lib.h

@@ -0,0 +1,107 @@
+/*
+ * hermann_lib.h - Ruby wrapper for the librdkafka library
+ *
+ * Copyright (c) 2014 Stan Campbell
+ * Copyright (c) 2014 Lookout, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *	this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *	this list of conditions and the following disclaimer in the documentation
+ *	and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HERMANN_H
+#define HERMANN_H
+
+#include <ruby.h>
+
+#include <ctype.h>
+#include <signal.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <syslog.h>
+#include <sys/time.h>
+#include <errno.h>
+
+#include <librdkafka/rdkafka.h>
+
+#ifdef TRACE
+#define TRACER(...) do {  \
+	fprintf(stderr, "%i:%s()> ", __LINE__, __PRETTY_FUNCTION__); \
+	fprintf(stderr, __VA_ARGS__); \
+	fflush(stderr); \
+					} while (0)
+#else
+#define TRACER(...) do { } while (0)
+#endif
+
+// Holds the defined Ruby module for Hermann
+static VALUE hermann_module;
+
+#define HERMANN_MAX_ERRSTR_LEN 512
+
+static int DEBUG = 0;
+
+// Should we expect rb_thread_blocking_region to be present?
+// #define RB_THREAD_BLOCKING_REGION
+#undef RB_THREAD_BLOCKING_REGION
+
+static 	enum {
+	OUTPUT_HEXDUMP,
+	OUTPUT_RAW,
+} output = OUTPUT_HEXDUMP;
+
+typedef struct HermannInstanceConfig {
+	char *topic;
+
+	/* Kafka configuration */
+	rd_kafka_t *rk;
+	rd_kafka_topic_t *rkt;
+	char *brokers;
+	int partition;
+	rd_kafka_topic_conf_t *topic_conf;
+	char errstr[512];
+	rd_kafka_conf_t *conf;
+	const char *debug;
+	int64_t start_offset;
+	int do_conf_dump;
+
+	int run;
+	int exit_eof;
+	int quiet;
+
+	int isInitialized;
+	int isConnected;
+
+	int isErrored;
+	char *error;
+} HermannInstanceConfig;
+
+typedef HermannInstanceConfig hermann_conf_t;
+
+typedef struct {
+	/* Hermann::Lib::Producer */
+	hermann_conf_t *producer;
+	/* Hermann::Result */
+	VALUE result;
+} hermann_push_ctx_t;
+
+#endif

data/ext/patches/librdkafka/0006-Update-some-headers-to-include-the-right-headers-to-.patch

@@ -0,0 +1,57 @@
+From 888ca33b571d99e877d665235b822f7c961c8fdb Mon Sep 17 00:00:00 2001
+From: "R. Tyler Croy" <tyler@monkeypox.org>
+Date: Thu, 28 Aug 2014 16:24:04 -0700
+Subject: [PATCH 6/8] Update some headers to include the right headers to build
+ on FreeBSD
+
+---
+ src/rd.h     | 9 +++++++++
+ src/rdaddr.h | 4 ++++
+ 2 files changed, 13 insertions(+)
+
+diff --git a/src/rd.h b/src/rd.h
+index c31501e..4789493 100644
+--- a/src/rd.h
++++ b/src/rd.h
+@@ -37,7 +37,11 @@
+ #include <errno.h>
+ #include <time.h>
+ #include <sys/time.h>
++
++#ifndef __FreeBSD__
++/* alloca(3) is in stdlib on FreeBSD */
+ #include <alloca.h>
++#endif
+ #include <assert.h>
+ #include <pthread.h>
+ 
+@@ -110,6 +114,11 @@
+ # endif
+ #endif /* sun */
+ 
++#ifdef __FreeBSD__
++/* FreeBSD defines be64toh() in sys/endian.h */
++#include <sys/endian.h>
++#endif
++
+ #ifndef be64toh
+ #ifndef __APPLE__
+ #ifndef sun
+diff --git a/src/rdaddr.h b/src/rdaddr.h
+index 0b37354..e55bd55 100644
+--- a/src/rdaddr.h
++++ b/src/rdaddr.h
+@@ -32,6 +32,10 @@
+ #include <arpa/inet.h>
+ #include <netdb.h>
+ 
++#ifdef __FreeBSD__
++#include <sys/socket.h>
++#endif
++
+ /**
+  * rd_sockaddr_inx_t is a union for either ipv4 or ipv6 sockaddrs.
+  * It provides conveniant abstraction of AF_INET* agnostic operations.
+-- 
+1.9.0
+

data/lib/hermann/consumer.rb

@@ -0,0 +1,19 @@
+require 'hermann'
+require 'hermann_lib'
+
+module Hermann
+  class Consumer
+    attr_reader :topic, :brokers, :partition, :internal
+
+    def initialize(topic, brokers, partition)
+      @topic = topic
+      @brokers = brokers
+      @partition = partition
+      @internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
+    end
+
+    def consume(&block)
+      @internal.consume(&block)
+    end
+  end
+end

data/lib/hermann/errors.rb

@@ -0,0 +1,8 @@
+
+module Hermann
+  module Errors
+    # Error for connectivity problems with the Kafka brokers
+    class ConnectivityError; end;
+  end
+end
+

data/lib/hermann/producer.rb

@@ -0,0 +1,122 @@
+require 'hermann'
+require 'hermann/result'
+require 'hermann_lib'
+
+module Hermann
+  class Producer
+    attr_reader :topic, :brokers, :internal, :children
+
+    def initialize(topic, brokers)
+      @topic = topic
+      @brokers = brokers
+      @internal = Hermann::Lib::Producer.new(topic, brokers)
+      # We're tracking children so we can make sure that at Producer exit we
+      # make a reasonable attempt to clean up outstanding result objects
+      @children = []
+    end
+
+    # @return [Boolean] True if our underlying producer object thinks it's
+    #   connected to a Kafka broker
+    def connected?
+      return @internal.connected?
+    end
+
+    # @return [Boolean] True if the underlying producer object has errored
+    def errored?
+      return @internal.errored?
+    end
+
+    def connect(timeout=0)
+      return @internal.connect(timeout * 1000)
+    end
+
+    # Push a value onto the Kafka topic passed to this +Producer+
+    #
+    # @param [Array] value An array of values to push, will push each one
+    #   separately
+    # @param [Object] value A single object to push
+    # @return [Hermann::Result] A future-like object which will store the
+    #   result from the broker
+    def push(value)
+      result = create_result
+
+      if value.kind_of? Array
+        return value.map { |e| self.push(e) }
+      else
+        @internal.push_single(value, result)
+      end
+
+      return result
+    end
+
+    # Create a +Hermann::Result+ that is tracked in the Producer's children
+    # array
+    #
+    # @return [Hermann::Result] A new, unused, result
+    def create_result
+      @children << Hermann::Result.new(self)
+      return @children.last
+    end
+
+    # Tick the underlying librdkafka reacter and clean up any unreaped but
+    # reapable children results
+    #
+    # @param [FixNum] timeout Seconds to block on the internal reactor
+    # @return [FixNum] Number of +Hermann::Result+ children reaped
+    def tick_reactor(timeout=0)
+      begin
+        execute_tick(rounded_timeout(timeout))
+      rescue StandardError => ex
+        @children.each do |child|
+          # Skip over any children that should already be reaped for other
+          # reasons
+          next if child.reap?
+          # Propagate errors to the remaining children
+          child.internal_set_error(ex)
+        end
+      end
+
+      # Reaping the children at this point will also reap any children marked
+      # as errored by an exception out of #execute_tick
+      return reap_children
+    end
+
+    # @return [FixNum] number of children reaped
+    def reap_children
+      # Filter all children who are no longer pending/fulfilled
+      total_children = @children.size
+      @children = @children.reject { |c| c.reap? }
+
+      return (total_children - children.size)
+    end
+
+
+    private
+
+    def rounded_timeout(timeout)
+      # Handle negative numbers, those can be zero
+      return 0 if (timeout < 0)
+      # Since we're going to sleep for each second, round any potential floats
+      # off
+      return timeout.round if timeout.kind_of?(Float)
+      return timeout
+    end
+
+    # Perform the actual reactor tick
+    # @raises [StandardError[ in case of underlying failures in librdkafka
+    def execute_tick(timeout)
+      if timeout == 0
+        @internal.tick(0)
+      else
+        (timeout * 2).times do
+          # We're going to Thread#sleep in Ruby to avoid a
+          # pthread_cond_timedwait(3) inside of librdkafka
+          events = @internal.tick(0)
+          # If we find events, break out early
+          break if events > 0
+          sleep 0.5
+        end
+      end
+    end
+  end
+end

data/lib/hermann/result.rb

@@ -0,0 +1,74 @@
+
+module Hermann
+  class Result
+    attr_reader :reason, :state
+
+    STATES = [:pending,
+              :rejected,
+              :fulfilled,
+              :unfulfilled,
+              ].freeze
+
+    def initialize(producer)
+      @producer = producer
+      @reason = nil
+      @value = nil
+      @state = :unfulfilled
+    end
+
+    STATES.each do |state|
+      define_method("#{state}?".to_sym) do
+        return @state == state
+      end
+    end
+
+    # @return [Boolean] True if this child can be reaped
+    def reap?
+      return true if rejected? || fulfilled?
+      return false
+    end
+
+    # Access the value of the future
+    #
+    # @param [FixNum] timeout Seconds to wait on the underlying machinery for a
+    #   result
+    # @return [NilClass] nil if no value could be received in the time alotted
+    # @return [Object]
+    def value(timeout=0)
+      @producer.tick_reactor(timeout)
+      return @value
+    end
+
+    # INTERNAL METHOD ONLY. Do not use
+    #
+    # This method will be invoked by the underlying extension to indicate set
+    # the actual value after a callback has completed
+    #
+    # @param [Object] value The actual resulting value
+    # @param [Boolean] is_error True if the result was errored for whatever
+    #   reason
+    def internal_set_value(value, is_error)
+      @value = value
+
+      if is_error
+        puts "Hermann::Result#set_internal_value(#{value.class}:\"#{value}\", error?:#{is_error})"
+        @state = :rejected
+      else
+        @state = :fulfilled
+      end
+    end
+
+    # INTERNAL METHOD ONLY. Do not use
+    #
+    # This method will set our internal #reason with the details from the
+    # exception
+    #
+    # @param [Exception] exception
+    def internal_set_error(exception)
+      return if exception.nil?
+
+      @reason = exception
+      @state = :rejected
+    end
+  end
+end

data/lib/hermann/timeout.rb

@@ -0,0 +1,37 @@
+begin
+  require 'system_timer'
+  module Hermann
+    class Timeout
+      USE_SYSTEM_TIMER = true
+    end
+  end
+rescue LoadError
+  require 'timeout'
+
+  if RUBY_VERSION == '1.8.7'
+    puts ">>> You are running on 1.8.7 without SystemTimer"
+    puts ">>> which means Hermann::Timeout will not work as expected"
+  end
+  module Hermann
+    class Timeout
+      USE_SYSTEM_TIMER = false
+    end
+  end
+end
+
+module Hermann
+  class Timeout
+    def self.system_timer?
+      Hermann::Timeout::USE_SYSTEM_TIMER
+    end
+
+    def self.timeout(seconds, klass=nil, &block)
+      if system_timer?
+        SystemTimer.timeout_after(seconds, klass, &block)
+      else
+        ::Timeout.timeout(seconds, klass, &block)
+      end
+    end
+  end
+end
+

data/lib/hermann/version.rb

@@ -0,0 +1,3 @@
+module Hermann
+  VERSION = '0.16.0'
+end

data/lib/hermann.rb

@@ -0,0 +1,2 @@
+module Hermann
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification 
+name: hermann
+version: !ruby/object:Gem::Version 
+  version: 0.16.0
+platform: ruby
+authors: 
+- Stan Campbell
+- R. Tyler Croy
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2014-09-12 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mini_portile
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *id001
+description: Ruby gem wrapper for librdkafka
+email: 
+- stan.campbell3@gmail.com
+- rtyler.croy@lookout.com
+executables: []
+
+extensions: 
+- ext/hermann/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- ext/hermann/extconf.rb
+- ext/hermann/hermann_lib.c
+- ext/hermann/hermann_lib.h
+- ext/patches/librdkafka/0006-Update-some-headers-to-include-the-right-headers-to-.patch
+- lib/hermann.rb
+- lib/hermann/consumer.rb
+- lib/hermann/errors.rb
+- lib/hermann/producer.rb
+- lib/hermann/result.rb
+- lib/hermann/timeout.rb
+- lib/hermann/version.rb
+homepage: https://github.com/lookout/Hermann
+licenses: 
+- MIT
+metadata: {}
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+- ext/hermann
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - &id002 
+    - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - *id002
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 3
+summary: A Kafka consumer/producer gem based on the librdkafka C library.
+test_files: []
+