heller 0.0.3-java → 0.2.0-java

data/spec/integration/end_to_end_communication_spec.rb

@@ -0,0 +1,316 @@
+# encoding: utf-8
+
+require 'spec_helper'
+
+
+describe 'End-to-end communication' do
+  let :producer do
+    Heller::Producer.new('localhost:9092', producer_options)
+  end
+
+  let :consumer do
+    Heller::Consumer.new('localhost:9092', client_id: 'spec-consumer')
+  end
+
+  let :producer_options do
+    {
+      client_id: 'spec-producer',
+      batch_size: 1,
+      num_retries: 10,
+      retry_backoff: 1000,
+    }
+  end
+
+  after do
+    producer.close
+    consumer.close
+  end
+
+  context 'Producer' do
+    context 'without an explicit key' do
+      let :topic do
+        "spec-without-explicit-key-#{Time.now.to_i.to_s(36)}"
+      end
+
+      it 'is able to push messages' do
+        expect { producer.push(Heller::Message.new(topic, 'simple string message')) }.not_to raise_error
+      end
+    end
+
+    context 'with an explicit key' do
+      let :topic do
+        "spec-with-explicit-key-#{Time.now.to_i.to_s(36)}"
+      end
+
+      it 'is able to push messages' do
+        expect { producer.push(Heller::Message.new(topic, 'simple string message', 'some-key')) }.not_to raise_error
+      end
+    end
+  end
+
+  context 'Consumer' do
+    describe '#fetch' do
+      let :fetch_response do
+        consumer.fetch(Heller::FetchRequest.new(topic, 0, 0))
+      end
+
+      let :enumerator do
+        fetch_response.messages(topic, 0)
+      end
+
+      context 'simple string messages' do
+        let :topic do
+          "spec-simple-string-#{Time.now.to_i.to_s(36)}"
+        end
+
+        before do
+          producer.push(Heller::Message.new(topic, 'simple string message'))
+        end
+
+        it 'is no big deal' do
+          expect(enumerator).to be_a(Heller::MessageSetEnumerator)
+
+          messages = enumerator.to_a
+          expect(messages.size).to eq(1)
+
+          offset, message = messages.last
+          expect(offset).to be_zero
+          expect(message).to eq('simple string message')
+        end
+      end
+
+      context 'JSON serialized hashes' do
+        let :topic do
+          "spec-json-hash-#{Time.now.to_i.to_s(36)}"
+        end
+
+        before do
+          producer.push(Heller::Message.new(topic, {'a key' => 'a value'}.to_json))
+        end
+
+        it 'is no big deal' do
+          expect(enumerator).to be_a(Heller::MessageSetEnumerator)
+
+          messages = enumerator.to_a
+          expect(messages.size).to eq(1)
+
+          offset, message = messages.last
+          expect(offset).to be_zero
+          expect(JSON.parse(message)).to eq({'a key' => 'a value'})
+        end
+      end
+
+      context 'with Snappy compression' do
+        let :producer_options do
+          super.merge({
+            client_id: 'spec-producer-snappy',
+            compression: :snappy,
+          })
+        end
+
+        context 'simple string messages' do
+          let :topic do
+            "spec-snappy-simple-string-#{Time.now.to_i.to_s(36)}"
+          end
+
+          before do
+            producer.push(Heller::Message.new(topic, 'simple string message'))
+          end
+
+          it 'is no big deal' do
+            expect(enumerator).to be_a(Heller::MessageSetEnumerator)
+
+            messages = enumerator.to_a
+            expect(messages.size).to eq(1)
+
+            offset, message = messages.last
+            expect(offset).to be_zero
+            expect(message).to eq('simple string message')
+          end
+        end
+
+        context 'JSON serialized hashes' do
+          let :topic do
+            "spec-snappy-json-hash-#{Time.now.to_i.to_s(36)}"
+          end
+
+          before do
+            producer.push(Heller::Message.new(topic, {'a key' => 'a value'}.to_json))
+          end
+
+          it 'is no big deal' do
+            expect(enumerator).to be_a(Heller::MessageSetEnumerator)
+
+            messages = enumerator.to_a
+            expect(messages.size).to eq(1)
+
+            offset, message = messages.last
+            expect(offset).to be_zero
+            expect(JSON.parse(message)).to eq({'a key' => 'a value'})
+          end
+        end
+      end
+    end
+
+    describe '#metadata' do
+      before do
+        topics.each { |topic| producer.push(Heller::Message.new(topic, 'metadata request message')) }
+      end
+
+      context 'when given a single topic' do
+        let :topics do
+          ["spec-single-metadata-topic-#{Time.now.to_i.to_s(36)}"]
+        end
+
+        it 'returns metadata about given topic' do
+          response = consumer.metadata(topics)
+          metadata = response.metadata
+
+          expect(metadata.size).to eq(1)
+
+          topic_metadata = metadata.first
+          expect(topic_metadata.topic).to eq(topics.first)
+        end
+      end
+
+      context 'when given several topics' do
+        let :topics do
+          [1, 2, 3].map { |i| "spec-multiple-metadata-topics-#{i}-#{Time.now.to_i.to_s(36)}" }
+        end
+
+        it 'returns metadata about given topics' do
+          response = consumer.metadata(topics)
+          metadata = response.metadata
+
+          expect(metadata.size).to eq(3)
+
+          topics.zip(metadata).each do |topic, metadata|
+            expect(metadata.topic).to eq(topic)
+          end
+        end
+      end
+
+      describe '#leader_for' do
+        let :topics do
+          [1, 2, 3].map { |i| "spec-metadata-leader-for-#{i}-#{Time.now.to_i.to_s(36)}" }
+        end
+
+        let :response do
+          consumer.metadata(topics)
+        end
+
+        context 'for existing topic-partition combinations' do
+          it 'returns the correct leader for each topic-partition combination' do
+            topics.each do |topic|
+              leader = response.leader_for(topic, 0)
+
+              expect(leader.connection_string).to match(/[a-z0-9\-\.]+:9092/i)
+            end
+          end
+        end
+
+        context 'for non-existing topic-partition combinations' do
+          it 'raises NoSuchTopicPartitionCombinationError' do
+            expect { response.leader_for('non-existent', 0) }.to raise_error(Heller::NoSuchTopicPartitionCombinationError)
+          end
+        end
+      end
+
+      describe '#isr_for' do
+        let :topics do
+          [1, 2, 3].map { |i| "spec-metadata-isr-for-#{i}-#{Time.now.to_i.to_s(36)}" }
+        end
+
+        let :response do
+          consumer.metadata(topics)
+        end
+
+        context 'for existing topic-partition combinations' do
+          it 'returns the correct in sync replicas for each topic-partition combination' do
+            topics.each do |topic|
+              isr = response.isr_for(topic, 0)
+
+              expect(isr.size).to eq(1)
+
+              replica = isr.first
+
+              expect(replica.connection_string).to match(/[a-z0-9\-\.]+:9092/i)
+            end
+          end
+        end
+
+        context 'for non-existing topic-partition combinations' do
+          it 'raises NoSuchTopicPartitionCombinationError' do
+            expect { response.isr_for('non-existent', 0) }.to raise_error(Heller::NoSuchTopicPartitionCombinationError)
+          end
+        end
+      end
+    end
+
+    describe '#offsets_before' do
+      let :topic do
+        "spec-offsets-before-#{Time.now.to_i.to_s(36)}"
+      end
+
+      let :requests do
+        Heller::OffsetRequest.new(topic, 0, (Time.now + 1).to_i * 1000, 3)
+      end
+
+      let :response do
+        consumer.offsets_before(requests)
+      end
+
+      before do
+        3.times { producer.push(Heller::Message.new(topic, 'offsets request message')) }
+      end
+
+      context 'for existing topic-partition combination(s)' do
+        it 'returns the expected offsets' do
+          offsets = response.offsets(topic, 0)
+
+          expect(offsets.size).to eq(2)
+          expect(offsets.first).to be > offsets.last
+        end
+      end
+
+      context 'for non-existing topic-partition combination(s)' do
+        it 'raises NoSuchTopicPartitionCombinationError' do
+          expect { response.offsets('non-existent', 0) }.to raise_error(Heller::NoSuchTopicPartitionCombinationError)
+        end
+      end
+    end
+
+    describe '#earliest_offset' do
+      let :topic do
+        "spec-earliest-offset-#{Time.now.to_i.to_s(36)}"
+      end
+
+      let :response do
+        consumer.earliest_offset(topic, 0)
+      end
+
+      before do
+        3.times { producer.push(Heller::Message.new(topic, 'offsets request message')) }
+      end
+
+      it 'returns the earliest offset' do
+        expect(response).to be_zero
+      end
+    end
+
+    describe '#latest_offset' do
+      let :topic do
+        "spec-latest-offset-#{Time.now.to_i.to_s(36)}"
+      end
+
+      before do
+        3.times { producer.push(Heller::Message.new(topic, 'offsets request message')) }
+      end
+
+      it 'returns the latest offset' do
+        response = consumer.latest_offset(topic, 0)
+        expect(response).to eq(3)
+      end
+    end
+  end
+end

data/spec/integration/zookeeper_consumer_spec.rb

@@ -0,0 +1,85 @@
+# encoding: utf-8
+
+require 'spec_helper'
+
+
+describe 'End-to-end communication using ZookeeperConsumer' do
+  let :producer do
+    Heller::Producer.new('localhost:9092', client_id: 'test-producer', batch_size: 1, num_retries: 10, retry_backoff: 1000)
+  end
+
+  let :consumer do
+    Heller::ZookeeperConsumer.new('localhost:2181', {
+      group_id: 'test',
+      auto_reset_offset: :smallest,
+      timeout: 1000,
+      fetch_max_wait: 1000,
+    })
+  end
+
+  let :decoder do
+    Kafka::Serializer::StringDecoder.new(nil)
+  end
+
+  let :consumed do
+    []
+  end
+
+  before do
+    messages.each do |message|
+      producer.push(Heller::Message.new(topic, message))
+    end
+  end
+
+  after do
+    producer.close
+    consumer.close
+  end
+
+  describe '#create_streams' do
+    let :topic do
+      "spec-create-streams-#{Time.now.to_i.to_s(36).upcase}"
+    end
+
+    let :messages do
+      %w[first-stream-message second-stream-message third-stream-message]
+    end
+
+    it 'can actually consume messages' do
+      streams_map = consumer.create_streams({topic => 1}, {key_decoder: decoder, value_decoder: decoder})
+      stream = streams_map[topic].first
+      iterator = stream.iterator
+
+      messages.size.times do
+        consumed << iterator.next.message
+      end
+
+      expect(consumed).to eq(messages)
+    end
+  end
+
+  describe '#create_streams_by_filter' do
+    let :topic do
+      "spec-with-filter-#{time_component}"
+    end
+
+    let :time_component do
+      Time.now.to_i.to_s(36).upcase
+    end
+
+    let :messages do
+      %w[first-filter-message second-filter-message third-filter-message]
+    end
+
+    it 'can actually consume messages' do
+      streams = consumer.create_streams_by_filter(".+-#{time_component}", 1, {key_decoder: decoder, value_decoder: decoder})
+      iterator = streams.first.iterator
+
+      messages.size.times do
+        consumed << iterator.next.message
+      end
+
+      expect(consumed).to eq(messages)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+
+require 'bundler/setup'
+require 'json'
+require 'spec/support/fakers'
+
+RSpec.configure do |config|
+  config.include(Fakers)
+  config.order = 'random'
+  config.raise_errors_for_deprecations!
+  config.before(:suite) do
+    unless java.lang.System.getProperty('log4j.configuration')
+      @log4j_config_file = Tempfile.new('silent-log4j.properties')
+      @log4j_config_file.puts('log4j.rootLogger=OFF')
+      @log4j_config_file.rewind
+      java.lang.System.setProperty('log4j.configuration', %(file://#{@log4j_config_file.path}))
+    end
+  end
+  config.after(:suite) do
+    if @log4j_config_file
+      @log4j_config_file.close
+      @log4j_config_file.unlink
+    end
+  end
+end
+
+require 'coveralls'
+require 'simplecov'
+
+if ENV.include?('TRAVIS')
+  Coveralls.wear!
+  SimpleCov.formatter = Coveralls::SimpleCov::Formatter
+end
+
+SimpleCov.start do
+  add_group 'Source', 'lib'
+  add_group 'Unit tests', 'spec/heller'
+  add_group 'Integration tests', 'spec/integration'
+end
+
+require 'heller'

data/spec/support/fakers.rb

@@ -0,0 +1,45 @@
+module Fakers
+  FakeMessageAndOffset = Struct.new(:message, :offset)
+  FakeMessage = Struct.new(:payload)
+
+  class FakePayload
+    def initialize(payload)
+      @payload = payload.to_java_bytes
+    end
+
+    def limit
+      @payload.size
+    end
+
+    def get(buffer)
+      @payload.each_with_index do |b, i|
+        buffer[i] = b
+      end
+    end
+  end
+
+  def create_fake_message(payload)
+    FakeMessage.new(FakePayload.new(payload))
+  end
+
+  FakeMessageSet = Struct.new(:iterator)
+
+  class FakeIterator < Struct.new(:messages)
+    def next
+      messages.shift
+    end
+
+    def has_next?
+      messages.any?
+    end
+  end
+
+  def create_fake_message_set(*payloads)
+    messages = payloads.each_with_index.map do |payload, index|
+      message = create_fake_message(payload)
+      FakeMessageAndOffset.new(message, index)
+    end
+
+    FakeMessageSet.new(FakeIterator.new(messages))
+  end
+end

metadata

@@ -1,67 +1,102 @@
 --- !ruby/object:Gem::Specification
 name: heller
 version: !ruby/object:Gem::Version
-  version: 0.0.3
-  prerelease:
+  version: 0.2.0
 platform: java
 authors:
 - Mathias Söderberg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-11-15 00:00:00.000000000 Z
+date: 2015-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: kafka-jars
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.7.2.2
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.7.2.2
-    none: false
+        version: 0.8.2
+  name: kafka-jars
   prerelease: false
   type: :runtime
-description: Attempts to make Kafka's Java API fit a bit better with Ruby
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+description: Attempts to make Kafka's Java API a bit more Rubyesque
 email:
-- mathias.soederberg@gmail.com
+- mths@sdrbrg.se
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- README.md
 - lib/heller.rb
-- lib/kafka.rb
+- lib/heller/configuration.rb
 - lib/heller/consumer.rb
+- lib/heller/consumer_configuration.rb
+- lib/heller/errors.rb
+- lib/heller/fetch_request.rb
+- lib/heller/fetch_response.rb
+- lib/heller/message.rb
+- lib/heller/message_set_enumerator.rb
+- lib/heller/offset_request.rb
+- lib/heller/offset_response.rb
 - lib/heller/producer.rb
-homepage: http://github.com/mthssdrbrg/heller
-licenses: []
+- lib/heller/producer_configuration.rb
+- lib/heller/topic_metadata_response.rb
+- lib/heller/version.rb
+- lib/heller/zookeeper_consumer.rb
+- lib/kafka.rb
+- spec/heller/consumer_configuration_spec.rb
+- spec/heller/consumer_spec.rb
+- spec/heller/fetch_response_spec.rb
+- spec/heller/message_set_enumerator_spec.rb
+- spec/heller/offset_response_spec.rb
+- spec/heller/producer_configuration_spec.rb
+- spec/heller/producer_spec.rb
+- spec/heller/topic_metadata_response_spec.rb
+- spec/heller/zookeeper_consumer_spec.rb
+- spec/integration/end_to_end_communication_spec.rb
+- spec/integration/zookeeper_consumer_spec.rb
+- spec/spec_helper.rb
+- spec/support/fakers.rb
+homepage: https://github.com/mthssdrbrg/heller
+licenses:
+- Apache License 2.0
+metadata: {}
 post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 requirements: []
-rubyforge_project: heller
-rubygems_version: 1.8.24
+rubyforge_project:
+rubygems_version: 2.4.6
 signing_key:
-specification_version: 3
+specification_version: 4
 summary: JRuby wrapper for Kafka
-test_files: []
-has_rdoc:
-...
+test_files:
+- spec/spec_helper.rb
+- spec/heller/consumer_configuration_spec.rb
+- spec/heller/consumer_spec.rb
+- spec/heller/fetch_response_spec.rb
+- spec/heller/message_set_enumerator_spec.rb
+- spec/heller/offset_response_spec.rb
+- spec/heller/producer_configuration_spec.rb
+- spec/heller/producer_spec.rb
+- spec/heller/topic_metadata_response_spec.rb
+- spec/heller/zookeeper_consumer_spec.rb
+- spec/integration/end_to_end_communication_spec.rb
+- spec/integration/zookeeper_consumer_spec.rb
+- spec/support/fakers.rb