heller 0.0.3-java → 0.2.0-java

data/spec/heller/producer_configuration_spec.rb

@@ -0,0 +1,178 @@
+# encoding: utf-8
+
+require 'spec_helper'
+
+module Heller
+  describe ProducerConfiguration do
+    it 'has sane defaults for running locally' do
+      configuration = described_class.new
+
+      expect(configuration[:brokers]).to eq('localhost:9092')
+      expect(configuration[:serializer]).to eq('kafka.serializer.StringEncoder')
+      expect(configuration[:ack]).to eq(-1)
+      expect(configuration[:type]).to eq(:sync)
+    end
+
+    context 'given hash with user-defined options' do
+      context 'with Symbol keys' do
+        it 'merges them with the defaults' do
+          configuration = described_class.new({
+            brokers: 'localhost:9092,localhost:9093',
+            serializer: 'kafka.serializer.DefaultEncoder',
+            batch_size: 1500
+          })
+
+          expect(configuration[:brokers]).to eq('localhost:9092,localhost:9093')
+          expect(configuration[:serializer]).to eq('kafka.serializer.DefaultEncoder')
+          expect(configuration[:batch_size]).to eq(1500)
+        end
+      end
+
+      context 'with String keys' do
+        it 'merges them with the defaults' do
+          configuration = described_class.new({
+            'brokers' => 'localhost:9092,localhost:9093',
+            'serializer' => 'kafka.serializer.DefaultEncoder',
+            'batch_size' => 1500
+          })
+
+          expect(configuration[:brokers]).to eq('localhost:9092,localhost:9093')
+          expect(configuration[:serializer]).to eq('kafka.serializer.DefaultEncoder')
+          expect(configuration[:batch_size]).to eq(1500)
+        end
+      end
+    end
+
+    context '#to_java' do
+      shared_examples_for 'a ProducerConfiguration' do
+        let :configuration do
+          described_class.new(options).to_java
+        end
+
+        it 'returns an instance of Kafka::Producer::ProducerConfig' do
+          expect(configuration).to be_a(Kafka::Producer::ProducerConfig)
+        end
+
+        it 'sets #broker_list' do
+          expect(configuration.broker_list).to eq('localhost:9092,localhost:9093')
+        end
+
+        it 'sets #request_required_acks' do
+          expect(configuration.request_required_acks).to eq(-1)
+        end
+
+        it 'sets #producer_type' do
+          expect(configuration.producer_type).to eq('async')
+        end
+
+        it 'sets serializer_class' do
+          expect(configuration.serializer_class).to eq('kafka.serializer.StringEncoder')
+        end
+
+        it 'sets #key_serializer_class' do
+          expect(configuration.key_serializer_class).to eq('kafka.serializer.DefaultEncoder')
+        end
+
+        it 'sets #partitioner_class' do
+          expect(configuration.partitioner_class).to eq('kafka.producer.DefaultPartitioner')
+        end
+
+        it 'sets #compression_codec' do
+          expect(configuration.compression_codec.name).to eq('gzip')
+        end
+
+        it 'sets #message_send_max_retries' do
+          expect(configuration.message_send_max_retries).to eq(5)
+        end
+
+        it 'sets #retry_backoff_ms' do
+          expect(configuration.retry_backoff_ms).to eq(1500)
+        end
+
+        it 'sets #topic_metadata_refresh_interval_ms' do
+          expect(configuration.topic_metadata_refresh_interval_ms).to eq(5000)
+        end
+
+        it 'sets #queue_buffering_max_ms' do
+          expect(configuration.queue_buffering_max_ms).to eq(1000 * 100)
+        end
+
+        it 'sets #queue_buffering_max_messages' do
+          expect(configuration.queue_buffering_max_messages).to eq(10000)
+        end
+
+        it 'sets #queue_enqueue_timeout_ms' do
+          expect(configuration.queue_enqueue_timeout_ms).to eq(1000)
+        end
+
+        it 'sets #batch_num_messages' do
+          expect(configuration.batch_num_messages).to eq(2000)
+        end
+
+        it 'sets #send_buffer_bytes' do
+          expect(configuration.send_buffer_bytes).to eq(1024 * 1000)
+        end
+
+        it 'sets #client_id' do
+          expect(configuration.client_id).to eq('spec-client')
+        end
+
+        it 'sets #request_timeout_ms' do
+          expect(configuration.request_timeout_ms).to eq(10000)
+        end
+      end
+
+      context 'with Symbol keys' do
+        let :options do
+          {
+            brokers: 'localhost:9092,localhost:9093',
+            type: :async,
+            serializer: 'kafka.serializer.StringEncoder',
+            key_serializer: 'kafka.serializer.DefaultEncoder',
+            partitioner: 'kafka.producer.DefaultPartitioner',
+            compression: :gzip,
+            num_retries: 5,
+            retry_backoff: 1500,
+            metadata_refresh_interval: 5000,
+            batch_size: 2000,
+            client_id: 'spec-client',
+            request_timeout: 10000,
+            buffer_limit: 100 * 100,
+            buffer_timeout: 1000 * 100,
+            enqueue_timeout: 1000,
+            socket_buffer: 1024 * 1000,
+            ack: -1
+          }
+        end
+
+        it_behaves_like 'a ProducerConfiguration'
+      end
+
+      context 'with String keys' do
+        let :options do
+          {
+            'brokers' => 'localhost:9092,localhost:9093',
+            'type' => :async,
+            'serializer' => 'kafka.serializer.StringEncoder',
+            'key_serializer' => 'kafka.serializer.DefaultEncoder',
+            'partitioner' => 'kafka.producer.DefaultPartitioner',
+            'compression' => :gzip,
+            'num_retries' => 5,
+            'retry_backoff' => 1500,
+            'metadata_refresh_interval' => 5000,
+            'batch_size' => 2000,
+            'client_id' => 'spec-client',
+            'request_timeout' => 10000,
+            'buffer_limit' => 100 * 100,
+            'buffer_timeout' => 1000 * 100,
+            'enqueue_timeout' => 1000,
+            'socket_buffer' => 1024 * 1000,
+            'ack' => -1
+          }
+        end
+
+        it_behaves_like 'a ProducerConfiguration'
+      end
+    end
+  end
+end

data/spec/heller/producer_spec.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+
+require 'spec_helper'
+
+module Heller
+  describe Producer do
+    let(:producer) do
+      described_class.new('localhost:9092', producer_impl: producer_impl)
+    end
+
+    let :producer_impl do
+      double(:producer_impl, new: producer_spy)
+    end
+
+    let :producer_spy do
+      double(:producer, send: nil, close: nil)
+    end
+
+    describe '#new' do
+      it 'converts options to a Kafka::Producer::ProducerConfig object' do
+        described_class.new('localhost:9092,localhost:9093', type: :async, producer_impl: producer_impl)
+
+        expect(producer_impl).to have_received(:new).with(instance_of(Kafka::Producer::ProducerConfig))
+      end
+    end
+
+    describe '#push' do
+      it 'wraps messages in a java.util.ArrayList' do
+        messages = [Heller::Message.new('topic', 'actual message', 'key!'), Heller::Message.new('topic2', 'payload')]
+
+        expect(producer_spy).to receive(:send) do |msgs|
+          expect(msgs).to be_a(java.util.ArrayList)
+          expect(msgs.to_a).to eq(messages)
+        end
+
+        producer.push(messages)
+      end
+
+      it 'allows sending a single message' do
+        message = Heller::Message.new('topic', 'actual message')
+
+        expect(producer_spy).to receive(:send) do |msgs|
+          expect(msgs.size).to eq(1)
+          expect(msgs.first).to eq(message)
+        end
+
+        producer.push(message)
+      end
+    end
+
+    describe '#disconnect' do
+      it 'calls #close on the underlying producer' do
+        producer.disconnect
+
+        expect(producer_spy).to have_received(:close)
+      end
+
+      it 'is aliased to #close' do
+        producer.close
+
+        expect(producer_spy).to have_received(:close)
+      end
+    end
+  end
+end

data/spec/heller/topic_metadata_response_spec.rb

@@ -0,0 +1,122 @@
+# encoding: utf-8
+
+require 'spec_helper'
+
+module Heller
+  describe TopicMetadataResponse do
+    let :response do
+      described_class.new(underlying)
+    end
+
+    let :underlying do
+      double(:topic_metadata_response, topics_metadata: [fake_topics_metadata])
+    end
+
+    let :fake_topics_metadata do
+      double(:topics_metadata)
+    end
+
+    describe '#each' do
+      let :fake_partition_metadata do
+        double(:partition_metadata)
+      end
+
+      before do
+        allow(fake_topics_metadata).to receive(:topic).and_return('spec')
+        allow(fake_topics_metadata).to receive(:partitions_metadata).and_return([fake_partition_metadata])
+      end
+
+      it 'yields topic and partition_metadata' do
+        yielded = []
+        response.each { |topic, meta| yielded << [topic, meta] }
+        expect(yielded.flatten).to eq(['spec', fake_partition_metadata])
+      end
+    end
+
+    describe '#metadata' do
+      before do
+        allow(underlying).to receive(:topics_metadata)
+      end
+
+      it 'returns #topics_metadata' do
+        response.metadata
+
+        expect(underlying).to have_received(:topics_metadata).once
+      end
+    end
+
+    describe '#leader_for' do
+      context 'given a topic-partition combination that exists' do
+        let :fake_partition_metadata do
+          double(:partition_metadata)
+        end
+
+        before do
+          allow(fake_topics_metadata).to receive(:topic).and_return('spec')
+          allow(fake_topics_metadata).to receive(:partitions_metadata).and_return([fake_partition_metadata])
+          allow(fake_partition_metadata).to receive(:partition_id).and_return(0)
+          allow(fake_partition_metadata).to receive(:leader).and_return('a non-nil value')
+        end
+
+        it 'returns the leader' do
+          expect(response.leader_for('spec', 0)).not_to be_nil
+
+          expect(fake_partition_metadata).to have_received(:leader)
+        end
+
+        it 'caches the result' do
+          2.times { expect(response.leader_for('spec', 0)).not_to be_nil }
+
+          expect(fake_partition_metadata).to have_received(:leader).once
+        end
+      end
+
+      context 'given a topic-partition combination that does not exist' do
+        before do
+          allow(fake_topics_metadata).to receive(:topic).and_return('not-spec')
+        end
+
+        it 'raises NoSuchTopicPartitionCombinationError' do
+          expect { response.leader_for('non-existent', 1) }.to raise_error(NoSuchTopicPartitionCombinationError)
+        end
+      end
+    end
+
+    describe '#isr_for' do
+      context 'given a topic-partition combination that exists' do
+        let :fake_partition_metadata do
+          double(:partition_metadata)
+        end
+
+        before do
+          allow(fake_topics_metadata).to receive(:topic).and_return('spec')
+          allow(fake_topics_metadata).to receive(:partitions_metadata).and_return([fake_partition_metadata])
+          allow(fake_partition_metadata).to receive(:partition_id).and_return(0)
+          allow(fake_partition_metadata).to receive(:isr).and_return(['a non-nil value'])
+        end
+
+        it 'returns in sync replicas' do
+          expect(response.isr_for('spec', 0)).not_to be_nil
+
+          expect(fake_partition_metadata).to have_received(:isr)
+        end
+
+        it 'caches the result' do
+          2.times { expect(response.isr_for('spec', 0)).not_to be_nil }
+
+          expect(fake_partition_metadata).to have_received(:isr).once
+        end
+      end
+
+      context 'given a topic-partition combination that does not exist' do
+        before do
+          allow(fake_topics_metadata).to receive(:topic).and_return('not-spec')
+        end
+
+        it 'raises NoSuchTopicPartitionCombinationError' do
+          expect { response.isr_for('non-existent', 1) }.to raise_error(NoSuchTopicPartitionCombinationError)
+        end
+      end
+    end
+  end
+end

data/spec/heller/zookeeper_consumer_spec.rb

@@ -0,0 +1,166 @@
+# encoding: utf-8
+
+require 'spec_helper'
+
+
+module Heller
+  describe ZookeeperConsumer do
+    let :consumer do
+      described_class.new('localhost:2181', options, consumer_impl)
+    end
+
+    let :options do
+      {
+        group_id: 'test',
+      }
+    end
+
+    let :consumer_impl do
+      double(:consumer_impl)
+    end
+
+    let :values do
+      []
+    end
+
+    before do
+      allow(consumer_impl).to receive(:createJavaConsumerConnector).and_return(consumer_impl)
+      allow(consumer_impl).to receive(:create_message_streams) do |hash, *args|
+        values.concat(hash.values)
+      end
+      allow(consumer_impl).to receive(:create_message_streams_by_filter)
+    end
+
+    describe '#initialize' do
+      it 'creates a JavaConsumerConnector' do
+        described_class.new('localhost:2181', options, consumer_impl)
+
+        expect(consumer_impl).to have_received(:createJavaConsumerConnector)
+      end
+
+      it 'includes ZooKeeper hosts in configuration' do
+        allow(consumer_impl).to receive(:createJavaConsumerConnector) do |config|
+          expect(config.zk_connect).to eq('localhost:2181')
+        end
+
+        described_class.new('localhost:2181', options, consumer_impl)
+      end
+    end
+
+    describe '#create_streams' do
+      context 'when given :key_decoder and :value_decoder' do
+        let :key_decoder do
+          double(:key_decoder)
+        end
+
+        let :value_decoder do
+          double(:value_decoder)
+        end
+
+        before do
+          consumer.create_streams({'topic1' => 2}, key_decoder: key_decoder, value_decoder: value_decoder)
+        end
+
+        it 'creates message streams with given key decoder' do
+          expect(consumer_impl).to have_received(:create_message_streams).with(anything, key_decoder, anything)
+        end
+
+        it 'creates message streams with given value decoder' do
+          expect(consumer_impl).to have_received(:create_message_streams).with(anything, anything, value_decoder)
+        end
+
+        it 'converts longs to integers' do
+          values.each do |value|
+            expect(value).to be_a(java.lang.Integer)
+          end
+        end
+      end
+
+      context 'when not given any options' do
+        before do
+          consumer.create_streams({'topic1' => 2})
+        end
+
+        it 'creates message streams' do
+          expect(consumer_impl).to have_received(:create_message_streams)
+        end
+
+        it 'converts longs to integers' do
+          values.each do |value|
+            expect(value).to be_a(java.lang.Integer)
+          end
+        end
+      end
+    end
+
+    describe '#create_streams_by_filter' do
+      let :key_decoder do
+        nil
+      end
+
+      let :value_decoder do
+        nil
+      end
+
+      before do
+        consumer.create_streams_by_filter('hello-world', 1, key_decoder: key_decoder, value_decoder: value_decoder)
+      end
+
+      it 'creates message streams' do
+        expect(consumer_impl).to have_received(:create_message_streams_by_filter)
+      end
+
+      it 'creates a Whitelist object from given filter' do
+        expect(consumer_impl).to have_received(:create_message_streams_by_filter).with(instance_of(Kafka::Consumer::Whitelist), 1)
+      end
+
+      context 'when given :key_decoder and :value_decoder' do
+        let :key_decoder do
+          double(:key_decoder)
+        end
+
+        let :value_decoder do
+          double(:value_decoder)
+        end
+
+        it 'creates message streams with given key decoder' do
+          expect(consumer_impl).to have_received(:create_message_streams_by_filter).with(anything, 1, key_decoder, anything)
+        end
+
+        it 'creates message streams with given value decoder' do
+          expect(consumer_impl).to have_received(:create_message_streams_by_filter).with(anything, 1, anything, value_decoder)
+        end
+      end
+    end
+
+    describe '#commit' do
+      before do
+        allow(consumer_impl).to receive(:commit_offsets)
+      end
+
+      it 'calls #commit_offsets' do
+        consumer.commit
+
+        expect(consumer_impl).to have_received(:commit_offsets)
+      end
+    end
+
+    describe '#close' do
+      before do
+        allow(consumer_impl).to receive(:shutdown)
+      end
+
+      it 'calls #shutdown' do
+        consumer.close
+
+        expect(consumer_impl).to have_received(:shutdown)
+      end
+
+      it 'is aliased as #shutdown' do
+        consumer.shutdown
+
+        expect(consumer_impl).to have_received(:shutdown)
+      end
+    end
+  end
+end