poseidon 0.0.1

data/spec/unit/protocol_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+include Protocol
+describe RequestCommon do
+  it "roundtrips" do
+    rc = RequestCommon.new(0,1,2,"client_id")
+
+    req_buffer = RequestBuffer.new
+    rc.write(req_buffer)
+
+    resp_buffer = ResponseBuffer.new(req_buffer.to_s)
+    rc_roundtrip = RequestCommon.read(resp_buffer)
+
+    expect(rc).to eq(rc_roundtrip)
+  end
+end
+
+describe MetadataRequest do
+  it "roundtrips" do
+    rc = RequestCommon.new(0,1,2,"client_id")
+    mr = MetadataRequest.new(rc, ["topic1","topic2"])
+
+    req_buffer = RequestBuffer.new
+    mr.write(req_buffer)
+
+    resp_buffer = ResponseBuffer.new(req_buffer.to_s)
+    mr_roundtrip = MetadataRequest.read(resp_buffer)
+
+    expect(mr).to eq(mr_roundtrip)
+  end
+end
+
+describe "objects with errors" do
+  it "returns objects that have errors" do
+    message_set = MessageSet.new
+    message_set << Message.new(:value => "value", :key => "key")
+    partition_fetch_response = PartitionFetchResponse.new(0, 5, 100, message_set)
+    topic_fetch_response = TopicFetchResponse.new('test_topic',
+                                                  [partition_fetch_response])
+    response = FetchResponse.new(stub('common'), [topic_fetch_response])
+
+    expect(response.objects_with_errors).to eq([partition_fetch_response])
+  end
+
+  it "raises error when asked" do
+    message_set = MessageSet.new
+    message_set << Message.new(:value => "value", :key => "key")
+    partition_fetch_response = PartitionFetchResponse.new(0, 5, 100, message_set)
+    topic_fetch_response = TopicFetchResponse.new('test_topic',
+                                                  [partition_fetch_response])
+    response = FetchResponse.new(stub('common'), [topic_fetch_response])
+
+    expect { response.raise_error_if_one_exists }.to raise_error
+  end
+end

data/spec/unit/sync_producer_spec.rb

@@ -0,0 +1,141 @@
+require 'spec_helper'
+
+describe SyncProducer do
+  describe "creation" do
+
+    it "sets correct defaults" do
+      sp = SyncProducer.new(nil,nil)
+      expect(sp.ack_timeout_ms).to eq(1500)
+      expect(sp.retry_backoff_ms).to eq(100)
+      expect(sp.metadata_refresh_interval_ms).to eq(600_000)
+      expect(sp.required_acks).to eq(0)
+      expect(sp.max_send_retries).to eq(3)
+    end
+
+    it "raises ArgumentError on unknown options" do
+      expect { SyncProducer.new(nil,nil,:unknown => true) }.to raise_error(ArgumentError)
+    end
+  end
+
+  # Fetches metadata
+
+  describe "sending" do
+    before(:each) do
+      Kernel.stub!(:sleep)
+
+      @broker_pool = stub('broker_pool').as_null_object
+      BrokerPool.stub!(:new).and_return(@broker_pool)
+
+      @cluster_metadata = stub('cluster_metadata', :last_refreshed_at => Time.now).as_null_object
+      ClusterMetadata.stub!(:new).and_return(@cluster_metadata)
+
+      @mbts = stub('messages_to_send', :needs_metadata? => false).as_null_object
+      MessagesToSend.stub!(:new).and_return(@mbts)
+    end
+
+    context "needs metadata" do
+      before(:each) do
+        @mbts.stub!(:needs_metadata?).and_return(true)
+      end
+
+      it "fetches metadata" do
+        @broker_pool.should_recieve(:fetch_metadata)
+        @sp = SyncProducer.new("test_client", [])
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+    end
+
+    context "there are messages to send" do
+      before(:each) do
+        @mbts.stub!(:messages_for_brokers).and_return([double('mfb').as_null_object])
+      end
+
+      it "sends messages" do
+        @broker_pool.should_recieve(:execute_api_call, :producer, anything, anything, anything)
+
+        @sp = SyncProducer.new("test_client", [])
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+    end
+
+    context "always fails" do
+      before(:each) do
+        @mbts.stub!(:all_sent?).and_return(false)
+        @sp = SyncProducer.new("test_client", [])
+      end
+
+      it "retries the correct number of times" do
+        @mbts.should_receive(:messages_for_brokers).exactly(4).times
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+
+      it "sleeps the correct amount between retries" do
+        Kernel.should_receive(:sleep).with(0.1).exactly(4).times
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+
+      it "refreshes metadata between retries" do
+        @cluster_metadata.should_receive(:update).exactly(4).times
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+
+      it "returns false" do
+        expect(@sp.send_messages([Message.new(:topic => "topic", :value => "value")])).to eq(false)
+      end
+    end
+
+    context "no retries" do
+      before(:each) do
+        @mbts.stub!(:all_sent?).and_return(false)
+        @sp = SyncProducer.new("test_client", [], max_send_retries: 0)
+      end
+
+      it "does not call sleep" do
+        Kernel.should_receive(:sleep).exactly(0).times
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+    end
+
+    context "succeeds on first attempt" do
+      before(:each) do
+        @mbts.stub!(:all_sent?).and_return(true)
+        @sp = SyncProducer.new("test_client", [])
+      end
+
+      it "returns true" do
+        expect(@sp.send_messages([Message.new(:topic => "topic", :value => "value")])).to eq(true)
+      end
+
+      it "does not sleep" do
+        Kernel.should_not_receive(:sleep)
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+
+      it "only attempts to send once" do
+        @mbts.should_receive(:messages_for_brokers).once
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+    end
+
+    context "succeeds on second attempt" do
+      before(:each) do
+        @mbts.stub!(:all_sent?).and_return(false, true)
+        @sp = SyncProducer.new("test_client", [])
+      end
+
+      it "returns true" do
+        expect(@sp.send_messages([Message.new(:topic => "topic", :value => "value")])).to eq(true)
+      end
+
+      it "sleeps once" do
+        Kernel.should_receive(:sleep).once
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+
+      it "attempts to send twice" do
+        @mbts.should_receive(:messages_for_brokers).twice
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+      end
+    end
+  end
+end

data/spec/unit/topic_metadata_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe TopicMetadata do
+  context "encoding" do
+    it "roundtrips" do
+      partition_metadata = PartitionMetadata.new(0, 0, 0, [0], [0])
+      partitions = [partition_metadata]
+      tm = TopicMetadata.new(TopicMetadataStruct.new(0, "topic", partitions))
+
+      request_buffer = RequestBuffer.new
+      tm.write(request_buffer)
+
+      response_buffer = ResponseBuffer.new(request_buffer.to_s)
+      expect(TopicMetadata.read(response_buffer)).to eq(tm)
+    end
+  end
+end

metadata

@@ -0,0 +1,206 @@
+--- !ruby/object:Gem::Specification
+name: poseidon
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Bob Potter
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: daemon_controller
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A Kafka (http://kafka.apache.org/) producer and consumer
+email:
+- bobby.potter@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- .yardopts
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- TODO.md
+- examples/consumer.rb
+- examples/producer.rb
+- lib/poseidon.rb
+- lib/poseidon/broker_pool.rb
+- lib/poseidon/cluster_metadata.rb
+- lib/poseidon/compressed_value.rb
+- lib/poseidon/compression.rb
+- lib/poseidon/compression/gzip_codec.rb
+- lib/poseidon/compression/snappy_codec.rb
+- lib/poseidon/connection.rb
+- lib/poseidon/fetched_message.rb
+- lib/poseidon/message.rb
+- lib/poseidon/message_conductor.rb
+- lib/poseidon/message_set.rb
+- lib/poseidon/message_to_send.rb
+- lib/poseidon/messages_for_broker.rb
+- lib/poseidon/messages_to_send.rb
+- lib/poseidon/messages_to_send_batch.rb
+- lib/poseidon/partition_consumer.rb
+- lib/poseidon/producer.rb
+- lib/poseidon/producer_compression_config.rb
+- lib/poseidon/protocol.rb
+- lib/poseidon/protocol/protocol_struct.rb
+- lib/poseidon/protocol/request_buffer.rb
+- lib/poseidon/protocol/response_buffer.rb
+- lib/poseidon/sync_producer.rb
+- lib/poseidon/topic_metadata.rb
+- lib/poseidon/version.rb
+- poseidon.gemspec
+- spec/bin/kafka-run-class.sh
+- spec/integration/multiple_brokers/round_robin_spec.rb
+- spec/integration/multiple_brokers/spec_helper.rb
+- spec/integration/simple/compression_spec.rb
+- spec/integration/simple/connection_spec.rb
+- spec/integration/simple/multiple_brokers_spec.rb
+- spec/integration/simple/simple_producer_and_consumer_spec.rb
+- spec/integration/simple/spec_helper.rb
+- spec/integration/simple/unavailable_broker_spec.rb
+- spec/spec_helper.rb
+- spec/test_cluster.rb
+- spec/unit/broker_pool_spec.rb
+- spec/unit/cluster_metadata_spec.rb
+- spec/unit/compression_spec.rb
+- spec/unit/connection_spec.rb
+- spec/unit/fetched_message_spec.rb
+- spec/unit/message_conductor_spec.rb
+- spec/unit/message_set_spec.rb
+- spec/unit/message_spec.rb
+- spec/unit/message_to_send_spec.rb
+- spec/unit/messages_for_broker_spec.rb
+- spec/unit/messages_to_send_batch_spec.rb
+- spec/unit/messages_to_send_spec.rb
+- spec/unit/partition_consumer_spec.rb
+- spec/unit/producer_compression_config_spec.rb
+- spec/unit/producer_spec.rb
+- spec/unit/protocol_spec.rb
+- spec/unit/sync_producer_spec.rb
+- spec/unit/topic_metadata_spec.rb
+homepage: https://github.com/bpot/poseidon
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Poseidon is a producer and consumer implementation for Kafka >= 0.8
+test_files:
+- spec/bin/kafka-run-class.sh
+- spec/integration/multiple_brokers/round_robin_spec.rb
+- spec/integration/multiple_brokers/spec_helper.rb
+- spec/integration/simple/compression_spec.rb
+- spec/integration/simple/connection_spec.rb
+- spec/integration/simple/multiple_brokers_spec.rb
+- spec/integration/simple/simple_producer_and_consumer_spec.rb
+- spec/integration/simple/spec_helper.rb
+- spec/integration/simple/unavailable_broker_spec.rb
+- spec/spec_helper.rb
+- spec/test_cluster.rb
+- spec/unit/broker_pool_spec.rb
+- spec/unit/cluster_metadata_spec.rb
+- spec/unit/compression_spec.rb
+- spec/unit/connection_spec.rb
+- spec/unit/fetched_message_spec.rb
+- spec/unit/message_conductor_spec.rb
+- spec/unit/message_set_spec.rb
+- spec/unit/message_spec.rb
+- spec/unit/message_to_send_spec.rb
+- spec/unit/messages_for_broker_spec.rb
+- spec/unit/messages_to_send_batch_spec.rb
+- spec/unit/messages_to_send_spec.rb
+- spec/unit/partition_consumer_spec.rb
+- spec/unit/producer_compression_config_spec.rb
+- spec/unit/producer_spec.rb
+- spec/unit/protocol_spec.rb
+- spec/unit/sync_producer_spec.rb
+- spec/unit/topic_metadata_spec.rb
+has_rdoc: