codeclimate-poseidon 0.0.8

data/spec/unit/sync_producer_spec.rb

@@ -0,0 +1,156 @@
+require 'spec_helper'
+
+RSpec.describe SyncProducer do
+  describe "creation" do
+
+    it "sets correct defaults" do
+      sp = SyncProducer.new(nil,nil)
+      expect(sp.ack_timeout_ms).to eq(1500)
+      expect(sp.retry_backoff_ms).to eq(100)
+      expect(sp.metadata_refresh_interval_ms).to eq(600_000)
+      expect(sp.required_acks).to eq(0)
+      expect(sp.max_send_retries).to eq(3)
+      expect(sp.socket_timeout_ms).to eq(10_000)
+    end
+
+    it "raises ArgumentError on unknown options" do
+      expect { SyncProducer.new(nil,nil,:unknown => true) }.to raise_error(ArgumentError)
+    end
+  end
+
+  # Fetches metadata
+
+  describe "sending" do
+    before(:each) do
+      allow(Kernel).to receive(:sleep)
+
+      @broker_pool = double('broker_pool').as_null_object
+      allow(BrokerPool).to receive(:new).and_return(@broker_pool)
+
+      @cluster_metadata = double('cluster_metadata', :last_refreshed_at => Time.now).as_null_object
+      allow(ClusterMetadata).to receive(:new).and_return(@cluster_metadata)
+
+      @mbts = double('messages_to_send', :needs_metadata? => false).as_null_object
+      allow(MessagesToSend).to receive(:new).and_return(@mbts)
+    end
+
+    context "needs metadata" do
+      before(:each) do
+        allow(@mbts).to receive(:needs_metadata?).and_return(true)
+      end
+
+      it "fetches metadata" do
+        @broker_pool.should_recieve(:fetch_metadata)
+
+        @sp = SyncProducer.new("test_client", [])
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue Errors::UnableToFetchMetadata
+      end
+    end
+
+    context "there are messages to send" do
+      before(:each) do
+        allow(@mbts).to receive(:messages_for_brokers).and_return([double('mfb').as_null_object])
+      end
+
+      it "sends messages" do
+        @broker_pool.should_recieve(:execute_api_call, :producer, anything, anything, anything)
+
+        @sp = SyncProducer.new("test_client", [])
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue StandardError
+      end
+    end
+
+    context "always fails" do
+      before(:each) do
+        allow(@mbts).to receive(:pending_messages?).and_return(true)
+        @sp = SyncProducer.new("test_client", [])
+      end
+
+      it "retries the correct number of times" do
+        expect(@mbts).to receive(:messages_for_brokers).exactly(4).times
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue StandardError
+      end
+
+      it "sleeps the correct amount between retries" do
+        expect(Kernel).to receive(:sleep).with(0.1).exactly(4).times
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue StandardError
+      end
+
+      it "resets and refreshes metadata between retries" do
+        # First check
+        expect(@mbts).to receive(:needs_metadata?).and_return(false).ordered
+
+        # After first failure
+        3.times do
+          expect(@cluster_metadata).to receive(:reset).ordered
+          expect(@broker_pool).to receive(:close).ordered
+          expect(@mbts).to receive(:needs_metadata?).and_return(true).ordered
+          expect(@cluster_metadata).to receive(:update).ordered
+          expect(@mbts).to receive(:needs_metadata?).and_return(false).ordered
+        end
+
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue StandardError
+      end
+
+      it "raises an exception" do
+        expect {
+          @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+        }.to raise_error
+      end
+    end
+
+    context "no retries" do
+      before(:each) do
+        allow(@mbts).to receive(:pending_messages?).and_return(true)
+        @sp = SyncProducer.new("test_client", [], max_send_retries: 0)
+      end
+
+      it "does not call sleep" do
+        expect(Kernel).to receive(:sleep).exactly(0).times
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue Errors::UnableToFetchMetadata
+      end
+    end
+
+    context "succeeds on first attempt" do
+      before(:each) do
+        allow(@mbts).to receive(:pending_messages?).and_return(false)
+        @sp = SyncProducer.new("test_client", [])
+      end
+
+      it "returns true" do
+        expect(@sp.send_messages([Message.new(:topic => "topic", :value => "value")])).to eq(true)
+      end
+
+      it "does not sleep" do
+        expect(Kernel).not_to receive(:sleep)
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+
+      it "only attempts to send once" do
+        expect(@mbts).to receive(:messages_for_brokers).once
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+    end
+
+    context "succeeds on second attempt" do
+      before(:each) do
+        allow(@mbts).to receive(:pending_messages?).and_return(true, false)
+        @sp = SyncProducer.new("test_client", [])
+      end
+
+      it "returns true" do
+        expect(@sp.send_messages([Message.new(:topic => "topic", :value => "value")])).to eq(true)
+      end
+
+      it "sleeps once" do
+        expect(Kernel).to receive(:sleep).once
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+
+      it "attempts to send twice" do
+        expect(@mbts).to receive(:messages_for_brokers).twice
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+    end
+  end
+end

data/spec/unit/topic_metadata_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+
+RSpec.describe TopicMetadata do
+  context "encoding" do
+    it "roundtrips" do
+      partition_metadata = Protocol::PartitionMetadata.new(0, 0, 0, [0], [0])
+      partitions = [partition_metadata]
+      tm = TopicMetadata.new(Protocol::TopicMetadataStruct.new(0, "topic", partitions))
+
+      request_buffer = Protocol::RequestBuffer.new
+      tm.write(request_buffer)
+
+      response_buffer = Protocol::ResponseBuffer.new(request_buffer.to_s)
+      expect(TopicMetadata.read(response_buffer)).to eq(tm)
+    end
+  end
+
+  it 'determines leader for a partition' do
+    partition_metadata = Protocol::PartitionMetadata.new(0, 0, 0, [0], [0])
+    partitions = [partition_metadata]
+    tm = TopicMetadata.new(Protocol::TopicMetadataStruct.new(0, "topic", partitions))
+
+    expect(tm.partition_leader(0)).to eq(0)
+  end
+
+  context "#available_partitions" do
+    it "includes when missing replicas" do
+      partition_metadata = Protocol::PartitionMetadata.new(9, 0, 0, [0], [0])
+      partitions = [partition_metadata]
+      tm = TopicMetadata.new(Protocol::TopicMetadataStruct.new(0, "topic", partitions))
+
+      expect(tm.available_partitions.length).to be(1)
+    end
+
+    it "ignores other errors" do
+      partition_metadata = Protocol::PartitionMetadata.new(2, 0, 0, [0], [0])
+      partitions = [partition_metadata]
+      tm = TopicMetadata.new(Protocol::TopicMetadataStruct.new(0, "topic", partitions))
+
+      expect(tm.available_partitions.length).to be(0)
+    end
+  end
+end

metadata

@@ -0,0 +1,225 @@
+--- !ruby/object:Gem::Specification
+name: codeclimate-poseidon
+version: !ruby/object:Gem::Version
+  version: 0.0.8
+platform: ruby
+authors:
+- Bob Potter
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: snappy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A Kafka (http://kafka.apache.org/) producer and consumer
+email:
+- bobby.potter@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- .yardopts
+- CHANGES.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- TODO.md
+- examples/consumer.rb
+- examples/producer.rb
+- lib/poseidon.rb
+- lib/poseidon/broker_pool.rb
+- lib/poseidon/cluster_metadata.rb
+- lib/poseidon/compressed_value.rb
+- lib/poseidon/compression.rb
+- lib/poseidon/compression/gzip_codec.rb
+- lib/poseidon/compression/snappy_codec.rb
+- lib/poseidon/connection.rb
+- lib/poseidon/fetched_message.rb
+- lib/poseidon/message.rb
+- lib/poseidon/message_conductor.rb
+- lib/poseidon/message_set.rb
+- lib/poseidon/message_to_send.rb
+- lib/poseidon/messages_for_broker.rb
+- lib/poseidon/messages_to_send.rb
+- lib/poseidon/messages_to_send_batch.rb
+- lib/poseidon/partition_consumer.rb
+- lib/poseidon/producer.rb
+- lib/poseidon/producer_compression_config.rb
+- lib/poseidon/protocol.rb
+- lib/poseidon/protocol/protocol_struct.rb
+- lib/poseidon/protocol/request_buffer.rb
+- lib/poseidon/protocol/response_buffer.rb
+- lib/poseidon/sync_producer.rb
+- lib/poseidon/topic_metadata.rb
+- lib/poseidon/version.rb
+- log/.gitkeep
+- poseidon.gemspec
+- spec/integration/multiple_brokers/consumer_spec.rb
+- spec/integration/multiple_brokers/metadata_failures_spec.rb
+- spec/integration/multiple_brokers/rebalance_spec.rb
+- spec/integration/multiple_brokers/round_robin_spec.rb
+- spec/integration/multiple_brokers/spec_helper.rb
+- spec/integration/simple/compression_spec.rb
+- spec/integration/simple/connection_spec.rb
+- spec/integration/simple/multiple_brokers_spec.rb
+- spec/integration/simple/simple_producer_and_consumer_spec.rb
+- spec/integration/simple/spec_helper.rb
+- spec/integration/simple/truncated_messages_spec.rb
+- spec/integration/simple/unavailable_broker_spec.rb
+- spec/spec_helper.rb
+- spec/test_cluster.rb
+- spec/unit/broker_pool_spec.rb
+- spec/unit/cluster_metadata_spec.rb
+- spec/unit/compression/gzip_codec_spec.rb
+- spec/unit/compression/snappy_codec_spec.rb
+- spec/unit/compression_spec.rb
+- spec/unit/connection_spec.rb
+- spec/unit/fetched_message_spec.rb
+- spec/unit/message_conductor_spec.rb
+- spec/unit/message_set_spec.rb
+- spec/unit/message_spec.rb
+- spec/unit/message_to_send_spec.rb
+- spec/unit/messages_for_broker_spec.rb
+- spec/unit/messages_to_send_batch_spec.rb
+- spec/unit/messages_to_send_spec.rb
+- spec/unit/partition_consumer_spec.rb
+- spec/unit/producer_compression_config_spec.rb
+- spec/unit/producer_spec.rb
+- spec/unit/protocol/request_buffer_spec.rb
+- spec/unit/protocol_spec.rb
+- spec/unit/sync_producer_spec.rb
+- spec/unit/topic_metadata_spec.rb
+homepage: https://github.com/bpot/poseidon
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Poseidon is a producer and consumer implementation for Kafka >= 0.8
+test_files:
+- spec/integration/multiple_brokers/consumer_spec.rb
+- spec/integration/multiple_brokers/metadata_failures_spec.rb
+- spec/integration/multiple_brokers/rebalance_spec.rb
+- spec/integration/multiple_brokers/round_robin_spec.rb
+- spec/integration/multiple_brokers/spec_helper.rb
+- spec/integration/simple/compression_spec.rb
+- spec/integration/simple/connection_spec.rb
+- spec/integration/simple/multiple_brokers_spec.rb
+- spec/integration/simple/simple_producer_and_consumer_spec.rb
+- spec/integration/simple/spec_helper.rb
+- spec/integration/simple/truncated_messages_spec.rb
+- spec/integration/simple/unavailable_broker_spec.rb
+- spec/spec_helper.rb
+- spec/test_cluster.rb
+- spec/unit/broker_pool_spec.rb
+- spec/unit/cluster_metadata_spec.rb
+- spec/unit/compression/gzip_codec_spec.rb
+- spec/unit/compression/snappy_codec_spec.rb
+- spec/unit/compression_spec.rb
+- spec/unit/connection_spec.rb
+- spec/unit/fetched_message_spec.rb
+- spec/unit/message_conductor_spec.rb
+- spec/unit/message_set_spec.rb
+- spec/unit/message_spec.rb
+- spec/unit/message_to_send_spec.rb
+- spec/unit/messages_for_broker_spec.rb
+- spec/unit/messages_to_send_batch_spec.rb
+- spec/unit/messages_to_send_spec.rb
+- spec/unit/partition_consumer_spec.rb
+- spec/unit/producer_compression_config_spec.rb
+- spec/unit/producer_spec.rb
+- spec/unit/protocol/request_buffer_spec.rb
+- spec/unit/protocol_spec.rb
+- spec/unit/sync_producer_spec.rb
+- spec/unit/topic_metadata_spec.rb
+has_rdoc: