heller 0.0.3-java → 0.2.0-java

data/lib/heller/producer_configuration.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+
+module Heller
+  class ProducerConfiguration < Configuration
+
+    protected
+
+    def key_mappings
+      @key_mappings ||= {
+        brokers: 'metadata.broker.list',
+        type: 'producer.type',
+        serializer: 'serializer.class',
+        key_serializer: 'key.serializer.class',
+        partitioner: 'partitioner.class',
+        compression: 'compression.codec',
+        compressed_topics: 'compressed.topics',
+        num_retries: 'message.send.max.retries',
+        retry_backoff: 'retry.backoff.ms',
+        metadata_refresh_interval: 'topic.metadata.refresh.interval.ms',
+        batch_size: 'batch.num.messages',
+        client_id: 'client.id',
+        request_timeout: 'request.timeout.ms',
+        buffer_limit: 'queue.buffering.max.messages',
+        buffer_timeout: 'queue.buffering.max.ms',
+        enqueue_timeout: 'queue.enqueue.timeout.ms',
+        socket_buffer: 'send.buffer.bytes',
+        ack: 'request.required.acks',
+      }.freeze
+    end
+
+    def defaults
+      {
+        brokers: 'localhost:9092',
+        type: :sync,
+        serializer: 'kafka.serializer.StringEncoder',
+        ack: -1
+      }
+    end
+
+    def kafka_config_class
+      Kafka::Producer::ProducerConfig
+    end
+  end
+end

data/lib/heller/topic_metadata_response.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+
+module Heller
+  class TopicMetadataResponse
+    include Enumerable
+
+    def initialize(underlying)
+      @underlying = underlying
+
+      @cache = Hash.new.tap do |h|
+        CACHES.each do |type|
+          h[type] = Hash.new({})
+        end
+      end
+    end
+
+    def each(&block)
+      metadata.each do |topic_metadata|
+        topic_metadata.partitions_metadata.each do |partition_metadata|
+          yield topic_metadata.topic, partition_metadata
+        end
+      end
+    end
+
+    def metadata
+      @underlying.topics_metadata
+    end
+
+    def leader_for(topic, partition)
+      with_cache(:leader, topic, partition)
+    end
+
+    def isr_for(topic, partition)
+      with_cache(:isr, topic, partition)
+    end
+    alias_method :in_sync_replicas_for, :isr_for
+
+    private
+
+    CACHES = [:leader, :isr].freeze
+
+    def with_cache(type, topic, partition)
+      return @cache[type][topic][partition] if @cache[type][topic][partition]
+
+      partition_metadata = locate_partition_metadata(topic, partition)
+
+      if partition_metadata
+        @cache[type][topic][partition] = partition_metadata.send(type)
+      else
+        raise NoSuchTopicPartitionCombinationError, "Cannot find (#{topic}:#{partition}) combination"
+      end
+    end
+
+    def locate_partition_metadata(topic, partition)
+      metadata.each do |tm|
+        if tm.topic == topic
+          tm.partitions_metadata.each do |pm|
+            return pm if pm.partition_id == partition
+          end
+        end
+      end
+
+      nil
+    end
+  end
+end

data/lib/heller/version.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+
+module Heller
+  VERSION = '0.2.0'.freeze
+end

data/lib/heller/zookeeper_consumer.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+
+module Heller
+  class ZookeeperConsumer
+    def initialize(zk_hosts, options, consumer_impl=Kafka::Consumer::Consumer)
+      @consumer = create_consumer(consumer_impl, options.merge(zk_connect: zk_hosts))
+    end
+
+    def create_streams(topic_count_map, options={})
+      if options[:key_decoder] && options[:value_decoder]
+        @consumer.create_message_streams(convert_longs(topic_count_map), *options.values_at(:key_decoder, :value_decoder))
+      else
+        @consumer.create_message_streams(convert_longs(topic_count_map))
+      end
+    end
+
+    def create_streams_by_filter(filter, num_streams, options={})
+      whitelist = Kafka::Consumer::Whitelist.new(filter)
+      if options[:key_decoder] && options[:value_decoder]
+        @consumer.create_message_streams_by_filter(whitelist, num_streams, *options.values_at(:key_decoder, :value_decoder)).to_a
+      else
+        @consumer.create_message_streams_by_filter(whitelist, num_streams).to_a
+      end
+    end
+
+    def commit
+      @consumer.commit_offsets
+    end
+
+    def close
+      @consumer.shutdown
+    end
+    alias_method :shutdown, :close
+
+    private
+
+    def convert_longs(hash)
+      hash.each_with_object({}) do |(k, v), acc|
+        acc[k] = v.to_java(:int)
+      end
+    end
+
+    def create_consumer(consumer_impl, options)
+      consumer_impl.createJavaConsumerConnector(ConsumerConfiguration.new(options).to_java)
+    end
+  end
+end

data/lib/heller.rb

@@ -2,9 +2,28 @@
 
 require 'kafka'
 
+
 module Heller
-	java_import 'java.util.ArrayList'
+  java_import 'java.util.ArrayList'
+  java_import 'java.util.Properties'
+  java_import 'java.lang.IllegalArgumentException'
+  java_import 'java.util.NoSuchElementException'
+
+  HellerError = Class.new(StandardError)
+  NoSuchTopicPartitionCombinationError = Class.new(HellerError)
 end
 
-require 'heller/consumer'
+require 'heller/configuration'
+require 'heller/consumer_configuration'
 require 'heller/producer'
+require 'heller/producer_configuration'
+require 'heller/errors'
+require 'heller/fetch_request'
+require 'heller/fetch_response'
+require 'heller/topic_metadata_response'
+require 'heller/message'
+require 'heller/message_set_enumerator'
+require 'heller/offset_request'
+require 'heller/offset_response'
+require 'heller/consumer'
+require 'heller/zookeeper_consumer'

data/lib/kafka.rb

@@ -1,32 +1,55 @@
 # encoding: utf-8
 
-require 'java'
+require 'slf4j-jars'
 require 'kafka-jars'
 
+
 module Kafka
-	module Api
-		java_import 'kafka.api.FetchRequest'
-		java_import 'kafka.api.MultiFetchRequest'
-		java_import 'kafka.api.MultiFetchResponse'
-	end
-
-	module Consumer
-		java_import 'kafka.javaapi.consumer.SimpleConsumer'
-	end
-
-	module Message
-		java_import 'kafka.message.Message'
-		java_import 'kafka.javaapi.message.MessageSet'
-		java_import 'kafka.javaapi.message.ByteBufferMessageSet'
-	end
-
-	module Producer
-		java_import 'kafka.producer.ProducerConfig'
-		java_import 'kafka.producer.SyncProducerConfig'
-
-		java_import 'kafka.javaapi.producer.ProducerData'
-
-		java_import 'kafka.javaapi.producer.SyncProducer'
-		java_import 'kafka.javaapi.producer.Producer'
-	end
-end
+  module Api
+    java_import 'kafka.api.FetchRequest'
+    java_import 'kafka.api.FetchRequestBuilder'
+    java_import 'kafka.api.PartitionOffsetRequestInfo'
+    java_import 'kafka.api.OffsetRequest'
+  end
+
+  module JavaApi
+    java_import 'kafka.javaapi.FetchResponse'
+    java_import 'kafka.javaapi.OffsetRequest'
+    java_import 'kafka.javaapi.OffsetResponse'
+    java_import 'kafka.javaapi.PartitionMetadata'
+    java_import 'kafka.javaapi.TopicMetadata'
+    java_import 'kafka.javaapi.TopicMetadataRequest'
+    java_import 'kafka.javaapi.TopicMetadataResponse'
+  end
+
+  module Common
+    java_import 'kafka.common.TopicAndPartition'
+  end
+
+  module Serializer
+    java_import 'kafka.serializer.StringEncoder'
+    java_import 'kafka.serializer.StringDecoder'
+  end
+
+  module Consumer
+    include_package 'kafka.consumer'
+    java_import 'kafka.javaapi.consumer.SimpleConsumer'
+  end
+
+  module Message
+    java_import 'kafka.message.Message'
+    java_import 'kafka.javaapi.message.MessageSet'
+    java_import 'kafka.javaapi.message.ByteBufferMessageSet'
+  end
+
+  module Producer
+    java_import 'kafka.javaapi.producer.Producer'
+    java_import 'kafka.producer.ProducerConfig'
+    java_import 'kafka.producer.SyncProducerConfig'
+    java_import 'kafka.producer.KeyedMessage'
+  end
+
+  module Errors
+    java_import 'kafka.common.ErrorMapping'
+  end
+end

data/spec/heller/consumer_configuration_spec.rb

@@ -0,0 +1,196 @@
+# encoding: utf-8
+
+require 'spec_helper'
+
+
+module Heller
+  describe ConsumerConfiguration do
+    let :configuration do
+      described_class.new(options).to_java
+    end
+
+    describe '#to_java' do
+      shared_examples_for 'a ConsumerConfiguration' do
+        it 'returns a Kafka::Consumer::ConsumerConfig object' do
+          expect(configuration).to be_a(Kafka::Consumer::ConsumerConfig)
+        end
+
+        it 'sets #auto_commit_enable' do
+          expect(configuration.auto_commit_enable).to be false
+        end
+
+        it 'sets #auto_commit_interval_ms' do
+          expect(configuration.auto_commit_interval_ms).to eq(1000)
+        end
+
+        it 'sets #auto_offset_reset' do
+          expect(configuration.auto_offset_reset).to eq('smallest')
+        end
+
+        it 'sets #client_id' do
+          expect(configuration.client_id).to eq('spec-client-id')
+        end
+
+        it 'sets #consumer_timeout_ms' do
+          expect(configuration.consumer_timeout_ms).to eq(30)
+        end
+
+        it 'sets #fetch_min_bytes' do
+          expect(configuration.fetch_min_bytes).to eq(12345)
+        end
+
+        it 'sets #fetch_message_max_bytes' do
+          expect(configuration.fetch_message_max_bytes).to eq(2048)
+        end
+
+        it 'sets #fetch_wait_max_ms' do
+          expect(configuration.fetch_wait_max_ms).to eq(54321)
+        end
+
+        it 'sets #group_id' do
+          expect(configuration.group_id).to eq('spec-group')
+        end
+
+        it 'sets #num_consumer_fetchers' do
+          expect(configuration.num_consumer_fetchers).to eq(10)
+        end
+
+        it 'sets #queued_max_messages' do
+          expect(configuration.queued_max_messages).to eq(1500)
+        end
+
+        it 'sets #rebalance_backoff_ms' do
+          expect(configuration.rebalance_backoff_ms).to eq(125)
+        end
+
+        it 'sets #rebalance_max_retries' do
+          expect(configuration.rebalance_max_retries).to eq(15)
+        end
+
+        it 'sets #socket_receive_buffer_bytes' do
+          expect(configuration.socket_receive_buffer_bytes).to eq(4096)
+        end
+
+        it 'sets #socket_timeout_ms' do
+          expect(configuration.socket_timeout_ms).to eq(100)
+        end
+
+        it 'sets #zk_connect' do
+          expect(configuration.zk_connect).to eq('localhost:2181')
+        end
+
+        it 'sets #zk_session_timeout_ms' do
+          expect(configuration.zk_session_timeout_ms).to eq(125)
+        end
+
+        it 'sets #zk_connection_timeout_ms' do
+          expect(configuration.zk_connection_timeout_ms).to eq(150)
+        end
+
+        it 'sets #zk_sync_time_ms' do
+          expect(configuration.zk_sync_time_ms).to eq(100)
+        end
+
+        it 'sets #partition_assignment_strategy' do
+          expect(configuration.partition_assignment_strategy).to eq('range')
+        end
+
+        it 'sets #offsets_storage' do
+          expect(configuration.offsets_storage).to eq('kafka')
+        end
+
+        it 'sets #offsets_channel_backoff_ms' do
+          expect(configuration.offsets_channel_backoff_ms).to eq(1000)
+        end
+
+        it 'sets #offsets_channel_socket_timeout_ms' do
+          expect(configuration.offsets_channel_socket_timeout_ms).to eq(1500)
+        end
+
+        it 'sets #offsets_commit_max_retries' do
+          expect(configuration.offsets_commit_max_retries).to eq(15)
+        end
+
+        it 'sets #dual_commit_enabled' do
+          expect(configuration.dual_commit_enabled).to be true
+        end
+
+        it 'sets #exclude_internal_topics' do
+          expect(configuration.exclude_internal_topics).to be false
+        end
+      end
+
+      context 'with Symbol keys' do
+        let :options do
+          {
+            auto_commit: false,
+            auto_commit_interval: 1000,
+            auto_reset_offset: :smallest,
+            client_id: 'spec-client-id',
+            fetch_message_max_bytes: 2048,
+            fetch_min_bytes: 12345,
+            fetch_max_wait: 54321,
+            group_id: 'spec-group',
+            num_fetchers: 10,
+            max_queued_message_chunks: 1500,
+            rebalance_retries: 15,
+            rebalance_retry_backoff: 125,
+            receive_buffer: 4096,
+            refresh_leader_backoff: 250,
+            socket_timeout: 100,
+            timeout: 30,
+            zk_connect: 'localhost:2181',
+            zk_session_timeout: 125,
+            zk_connection_timeout: 150,
+            zk_sync_time: 100,
+            partition_assignment_strategy: 'range',
+            offsets_storage: 'kafka',
+            offsets_channel_backoff_ms: 1000,
+            offsets_channel_socket_timeout_ms: 1500,
+            offsets_commit_max_retries: 15,
+            dual_commit_enabled: true,
+            exclude_internal_topics: false,
+          }
+        end
+
+        it_behaves_like 'a ConsumerConfiguration'
+      end
+
+      context 'when given String keys' do
+        let :options do
+          {
+            'auto_commit' => false,
+            'auto_commit_interval' => 1000,
+            'auto_reset_offset' => :smallest,
+            'client_id' => 'spec-client-id',
+            'fetch_message_max_bytes' => 2048,
+            'fetch_min_bytes' => 12345,
+            'fetch_max_wait' => 54321,
+            'group_id' => 'spec-group',
+            'num_fetchers' => 10,
+            'max_queued_message_chunks' => 1500,
+            'rebalance_retries' => 15,
+            'rebalance_retry_backoff' => 125,
+            'receive_buffer' => 4096,
+            'refresh_leader_backoff' => 250,
+            'socket_timeout' => 100,
+            'timeout' => 30,
+            'zk_connect' => 'localhost:2181',
+            'zk_session_timeout' => 125,
+            'zk_connection_timeout' => 150,
+            'zk_sync_time' => 100,
+            'partition_assignment_strategy' => 'range',
+            'offsets_storage' => 'kafka',
+            'offsets_channel_backoff_ms' => 1000,
+            'offsets_channel_socket_timeout_ms' => 1500,
+            'offsets_commit_max_retries' => 15,
+            'dual_commit_enabled' => true,
+            'exclude_internal_topics' => false,
+          }
+        end
+
+        it_behaves_like 'a ConsumerConfiguration'
+      end
+    end
+  end
+end