hermann 0.11-x86-darwin-12 → 0.15-x86-darwin-12

data/ext/{hermann_lib.h → hermann/hermann_lib.h}

@@ -2,16 +2,17 @@
  * hermann_lib.h - Ruby wrapper for the librdkafka library
  *
  * Copyright (c) 2014 Stan Campbell
+ * Copyright (c) 2014 Lookout, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  * 1. Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
+ *	this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
+ *	this list of conditions and the following disclaimer in the documentation
+ *	and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -42,8 +43,21 @@
 
 #include <librdkafka/rdkafka.h>
 
+#ifdef TRACE
+#define TRACER(...) do {  \
+	fprintf(stderr, "%i:%s()> ", __LINE__, __PRETTY_FUNCTION__); \
+	fprintf(stderr, __VA_ARGS__); \
+	fflush(stderr); \
+					} while (0)
+#else
+#define TRACER(...) do { } while (0)
+#endif
+
 // Holds the defined Ruby module for Hermann
-static VALUE m_hermann;
+static VALUE hermann_module;
+
+#define HERMANN_MAX_ERRSTR_LEN 512
+#define HERMANN_MAX_TOPIC_LEN 512
 
 static int DEBUG = 0;
 
@@ -57,27 +71,38 @@ static 	enum {
 } output = OUTPUT_HEXDUMP;
 
 typedef struct HermannInstanceConfig {
+	char *topic;
 
-    char* topic;
-
-    /* Kafka configuration */
-    rd_kafka_t *rk;
-    rd_kafka_topic_t *rkt;
-    char *brokers;
-    int partition;
-    rd_kafka_topic_conf_t *topic_conf;
-    char errstr[512];
-    rd_kafka_conf_t *conf;
-    const char *debug;
-    int64_t start_offset;
-    int do_conf_dump;
+	/* Kafka configuration */
+	rd_kafka_t *rk;
+	rd_kafka_topic_t *rkt;
+	char *brokers;
+	int partition;
+	rd_kafka_topic_conf_t *topic_conf;
+	char errstr[512];
+	rd_kafka_conf_t *conf;
+	const char *debug;
+	int64_t start_offset;
+	int do_conf_dump;
 
-    int run;
-    int exit_eof;
-    int quiet;
+	int run;
+	int exit_eof;
+	int quiet;
 
-    int isInitialized;
+	int isInitialized;
+	int isConnected;
 
+	int isErrored;
+	char *error;
 } HermannInstanceConfig;
 
-#endif
+typedef HermannInstanceConfig hermann_conf_t;
+
+typedef struct {
+	/* Hermann::Lib::Producer */
+	hermann_conf_t *producer;
+	/* Hermann::Result */
+	VALUE result;
+} hermann_push_ctx_t;
+
+#endif

data/lib/hermann/consumer.rb

@@ -0,0 +1,19 @@
+require 'hermann'
+require 'hermann_lib'
+
+module Hermann
+  class Consumer
+    attr_reader :topic, :brokers, :partition, :internal
+
+    def initialize(topic, brokers, partition)
+      @topic = topic
+      @brokers = brokers
+      @partition = partition
+      @internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
+    end
+
+    def consume(&block)
+      @internal.consume(&block)
+    end
+  end
+end

data/lib/hermann/errors.rb

@@ -0,0 +1,8 @@
+
+module Hermann
+  module Errors
+    # Error for connectivity problems with the Kafka brokers
+    class ConnectivityError; end;
+  end
+end
+

data/lib/hermann/producer.rb

@@ -0,0 +1,122 @@
+require 'hermann'
+require 'hermann/result'
+require 'hermann_lib'
+
+module Hermann
+  class Producer
+    attr_reader :topic, :brokers, :internal, :children
+
+    def initialize(topic, brokers)
+      @topic = topic
+      @brokers = brokers
+      @internal = Hermann::Lib::Producer.new(topic, brokers)
+      # We're tracking children so we can make sure that at Producer exit we
+      # make a reasonable attempt to clean up outstanding result objects
+      @children = []
+    end
+
+    # @return [Boolean] True if our underlying producer object thinks it's
+    #   connected to a Kafka broker
+    def connected?
+      return @internal.connected?
+    end
+
+    # @return [Boolean] True if the underlying producer object has errored
+    def errored?
+      return @internal.errored?
+    end
+
+    def connect(timeout=0)
+      return @internal.connect(timeout * 1000)
+    end
+
+    # Push a value onto the Kafka topic passed to this +Producer+
+    #
+    # @param [Array] value An array of values to push, will push each one
+    #   separately
+    # @param [Object] value A single object to push
+    # @return [Hermann::Result] A future-like object which will store the
+    #   result from the broker
+    def push(value)
+      result = create_result
+
+      if value.kind_of? Array
+        return value.map { |e| self.push(e) }
+      else
+        @internal.push_single(value, result)
+      end
+
+      return result
+    end
+
+    # Create a +Hermann::Result+ that is tracked in the Producer's children
+    # array
+    #
+    # @return [Hermann::Result] A new, unused, result
+    def create_result
+      @children << Hermann::Result.new(self)
+      return @children.last
+    end
+
+    # Tick the underlying librdkafka reacter and clean up any unreaped but
+    # reapable children results
+    #
+    # @param [FixNum] timeout Seconds to block on the internal reactor
+    # @return [FixNum] Number of +Hermann::Result+ children reaped
+    def tick_reactor(timeout=0)
+      begin
+        execute_tick(rounded_timeout(timeout))
+      rescue StandardError => ex
+        @children.each do |child|
+          # Skip over any children that should already be reaped for other
+          # reasons
+          next if child.reap?
+          # Propagate errors to the remaining children
+          child.internal_set_error(ex)
+        end
+      end
+
+      # Reaping the children at this point will also reap any children marked
+      # as errored by an exception out of #execute_tick
+      return reap_children
+    end
+
+    # @return [FixNum] number of children reaped
+    def reap_children
+      # Filter all children who are no longer pending/fulfilled
+      total_children = @children.size
+      @children = @children.reject { |c| c.reap? }
+
+      return (total_children - children.size)
+    end
+
+
+    private
+
+    def rounded_timeout(timeout)
+      # Handle negative numbers, those can be zero
+      return 0 if (timeout < 0)
+      # Since we're going to sleep for each second, round any potential floats
+      # off
+      return timeout.round if timeout.kind_of?(Float)
+      return timeout
+    end
+
+    # Perform the actual reactor tick
+    # @raises [StandardError[ in case of underlying failures in librdkafka
+    def execute_tick(timeout)
+      if timeout == 0
+        @internal.tick(0)
+      else
+        (timeout * 2).times do
+          # We're going to Thread#sleep in Ruby to avoid a
+          # pthread_cond_timedwait(3) inside of librdkafka
+          events = @internal.tick(0)
+          # If we find events, break out early
+          break if events > 0
+          sleep 0.5
+        end
+      end
+    end
+  end
+end

data/lib/hermann/result.rb

@@ -0,0 +1,74 @@
+
+module Hermann
+  class Result
+    attr_reader :reason, :state
+
+    STATES = [:pending,
+              :rejected,
+              :fulfilled,
+              :unfulfilled,
+              ].freeze
+
+    def initialize(producer)
+      @producer = producer
+      @reason = nil
+      @value = nil
+      @state = :unfulfilled
+    end
+
+    STATES.each do |state|
+      define_method("#{state}?".to_sym) do
+        return @state == state
+      end
+    end
+
+    # @return [Boolean] True if this child can be reaped
+    def reap?
+      return true if rejected? || fulfilled?
+      return false
+    end
+
+    # Access the value of the future
+    #
+    # @param [FixNum] timeout Seconds to wait on the underlying machinery for a
+    #   result
+    # @return [NilClass] nil if no value could be received in the time alotted
+    # @return [Object]
+    def value(timeout=0)
+      @producer.tick_reactor(timeout)
+      return @value
+    end
+
+    # INTERNAL METHOD ONLY. Do not use
+    #
+    # This method will be invoked by the underlying extension to indicate set
+    # the actual value after a callback has completed
+    #
+    # @param [Object] value The actual resulting value
+    # @param [Boolean] is_error True if the result was errored for whatever
+    #   reason
+    def internal_set_value(value, is_error)
+      @value = value
+
+      if is_error
+        puts "Hermann::Result#set_internal_value(#{value.class}:\"#{value}\", error?:#{is_error})"
+        @state = :rejected
+      else
+        @state = :fulfilled
+      end
+    end
+
+    # INTERNAL METHOD ONLY. Do not use
+    #
+    # This method will set our internal #reason with the details from the
+    # exception
+    #
+    # @param [Exception] exception
+    def internal_set_error(exception)
+      return if exception.nil?
+
+      @reason = exception
+      @state = :rejected
+    end
+  end
+end

data/lib/hermann/timeout.rb

@@ -0,0 +1,37 @@
+begin
+  require 'system_timer'
+  module Hermann
+    class Timeout
+      USE_SYSTEM_TIMER = true
+    end
+  end
+rescue LoadError
+  require 'timeout'
+
+  if RUBY_VERSION == '1.8.7'
+    puts ">>> You are running on 1.8.7 without SystemTimer"
+    puts ">>> which means Hermann::Timeout will not work as expected"
+  end
+  module Hermann
+    class Timeout
+      USE_SYSTEM_TIMER = false
+    end
+  end
+end
+
+module Hermann
+  class Timeout
+    def self.system_timer?
+      Hermann::Timeout::USE_SYSTEM_TIMER
+    end
+
+    def self.timeout(seconds, klass=nil, &block)
+      if system_timer?
+        SystemTimer.timeout_after(seconds, klass, &block)
+      else
+        ::Timeout.timeout(seconds, klass, &block)
+      end
+    end
+  end
+end
+

data/lib/hermann/version.rb

@@ -0,0 +1,3 @@
+module Hermann
+  VERSION = '0.15'
+end

data/lib/hermann.rb

@@ -1 +1,2 @@
-require 'hermann_lib'
+module Hermann
+end

metadata

@@ -1,40 +1,47 @@
 --- !ruby/object:Gem::Specification 
 name: hermann
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 21
   prerelease: 
   segments: 
   - 0
-  - 11
-  version: "0.11"
+  - 15
+  version: "0.15"
 platform: x86-darwin-12
 authors: 
 - Stan Campbell
+- R. Tyler Croy
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2014-05-29 00:00:00 Z
+date: 2014-09-11 00:00:00 Z
 dependencies: []
 
-description: Ruby gem wrapper for the RdKafka C library
-email: stan.campbell3@gmail.com
+description: Ruby gem wrapper for librdkafka
+email: 
+- stan.campbell3@gmail.com
+- rtyler.croy@lookout.com
 executables: []
 
 extensions: 
-- ext/extconf.rb
+- ext/hermann/extconf.rb
 extra_rdoc_files: []
 
 files: 
 - Rakefile
-- ext/hermann_lib.h
-- ext/hermann_lib.c
-- ext/extconf.rb
-- bin/hermann
 - lib/hermann.rb
-- ext/hermann_lib.o
-- test/test_hermann.rb
-homepage: http://rubygems.org/gems/hermann
+- lib/hermann/consumer.rb
+- lib/hermann/errors.rb
+- lib/hermann/producer.rb
+- lib/hermann/result.rb
+- lib/hermann/timeout.rb
+- lib/hermann/version.rb
+- ext/.ruby-version
+- ext/hermann/extconf.rb
+- ext/hermann/hermann_lib.c
+- ext/hermann/hermann_lib.h
+homepage: https://github.com/lookout/Hermann
 licenses: 
 - MIT
 post_install_message: 
@@ -42,7 +49,7 @@ rdoc_options: []
 
 require_paths: 
 - lib
-- ext
+- ext/hermann
 required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
@@ -67,6 +74,6 @@ rubyforge_project:
 rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
-summary: The Kafka consumer is based on the librdkafka C library.
-test_files: 
-- test/test_hermann.rb
+summary: A Kafka consumer/producer gem based on the librdkafka C library.
+test_files: []
+

data/bin/hermann

@@ -1,4 +0,0 @@
-#!ruby
-
-require 'hermann'
-puts "Placeholder..."