poseidon 0.0.4 → 0.0.5.pre1

data/spec/unit/messages_to_send_spec.rb

@@ -38,12 +38,12 @@ describe MessagesToSend do
     context "is successful" do
       before(:each) do
         @mts.messages_for_brokers(nil).each do |mfb|
-          @mts.successfully_sent(mfb)
+          @mts.successfully_sent(mfb.messages)
         end
       end
 
       it "successfully sends all" do
-        expect(@mts.all_sent?).to eq(true)
+        expect(@mts.pending_messages?).to eq(false)
       end
     end
 
@@ -56,7 +56,7 @@ describe MessagesToSend do
       it "does not send all" do
         @mts.messages_for_brokers(nil).each do |mfb|
         end
-        expect(@mts.all_sent?).to eq(false)
+        expect(@mts.pending_messages?).to eq(true)
       end
     end
   end

data/spec/unit/partition_consumer_spec.rb

@@ -14,13 +14,13 @@ describe PartitionConsumer do
   describe "creation" do
     context "when passed unknown options" do
       it "raises an ArgumentError" do
-        expect { PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0,-2, :unknown => true) }.to raise_error(ArgumentError)
+        expect { PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :earliest_offset, :unknown => true) }.to raise_error(ArgumentError)
       end
     end
 
     context "when passed an unknown offset" do
       it "raises an ArgumentError" do
-        expect { PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0,:coolest_offset) }.to raise_error(ArgumentError)
+        expect { PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :coolest_offset) }.to raise_error(ArgumentError)
       end
     end
   end
@@ -30,14 +30,14 @@ describe PartitionConsumer do
       it "resolves offset if it's not set" do
         @connection.should_receive(:offset).and_return(@offset_response)
         pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
-                                   0, -2)
+                                   0, :earliest_offset)
 
         pc.next_offset
       end
 
       it "returns resolved offset" do
         pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
-                                   0, -2)
+                                   0, :earliest_offset)
         expect(pc.next_offset).to eq(100)
       end
     end
@@ -54,7 +54,7 @@ describe PartitionConsumer do
       it "is raised" do
         @offset_response.first.partition_offsets.first.stub!(:error).and_return(2)
         pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
-                                   0, -2)
+                                   0, :earliest_offset)
 
         expect { pc.next_offset }.to raise_error(Errors::InvalidMessage)
       end
@@ -63,12 +63,20 @@ describe PartitionConsumer do
     context "when no offset exists" do
       it "sets offset to 0" do
         pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
-                                   0, -2)
+                                   0, :earliest_offset)
 
         @offset_response.first.partition_offsets.first.stub!(:offsets).and_return([])
         expect(pc.next_offset).to eq(0)
       end
     end
+
+    context "when offset negative" do
+      it "resolves offset to one " do
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, -10)
+        expect(pc.next_offset).to eq(90)
+      end
+    end
   end
 
   describe "fetching messages" do
@@ -81,7 +89,7 @@ describe PartitionConsumer do
       @response = Protocol::FetchResponse.new(stub('common'), [topic_fetch_response])
 
       @connection.stub(:fetch).and_return(@response)
-      @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, -2)
+      @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :earliest_offset)
     end
 
     it "returns FetchedMessage objects" do
@@ -89,14 +97,24 @@ describe PartitionConsumer do
     end
 
     it "uses object defaults" do
-      @connection.should_receive(:fetch).with(10_000, 0, anything)
+      @connection.should_receive(:fetch).with(10_000, 1, anything)
       @pc.fetch
     end
 
     context "when options are passed" do
       it "overrides object defaults" do
-        @connection.should_receive(:fetch).with(20_000, 0, anything)
-        @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, -2, :max_wait_ms => 20_000)
+        @connection.should_receive(:fetch).with(20_000, 1, anything)
+        @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :earliest_offset, :max_wait_ms => 20_000)
+
+        @pc.fetch
+      end
+    end
+
+    context "when negative offset beyond beginning of partition is passed" do
+      it "starts from the earliest offset" do
+        @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, -10000)
+        pfr = @response.topic_fetch_responses.first.partition_fetch_responses.first
+        pfr.stub!(:error).and_return(1, 1, 0)
 
         @pc.fetch
       end

data/spec/unit/sync_producer_spec.rb

@@ -10,6 +10,7 @@ describe SyncProducer do
       expect(sp.metadata_refresh_interval_ms).to eq(600_000)
       expect(sp.required_acks).to eq(0)
       expect(sp.max_send_retries).to eq(3)
+      expect(sp.socket_timeout_ms).to eq(10_000)
     end
 
     it "raises ArgumentError on unknown options" do
@@ -40,8 +41,9 @@ describe SyncProducer do
 
       it "fetches metadata" do
         @broker_pool.should_recieve(:fetch_metadata)
+
         @sp = SyncProducer.new("test_client", [])
-        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue Errors::UnableToFetchMetadata
       end
     end
 
@@ -54,51 +56,53 @@ describe SyncProducer do
         @broker_pool.should_recieve(:execute_api_call, :producer, anything, anything, anything)
 
         @sp = SyncProducer.new("test_client", [])
-        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue StandardError
       end
     end
 
     context "always fails" do
       before(:each) do
-        @mbts.stub!(:all_sent?).and_return(false)
+        @mbts.stub!(:pending_messages?).and_return(true)
         @sp = SyncProducer.new("test_client", [])
       end
 
       it "retries the correct number of times" do
         @mbts.should_receive(:messages_for_brokers).exactly(4).times
-        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue StandardError
       end
 
       it "sleeps the correct amount between retries" do
         Kernel.should_receive(:sleep).with(0.1).exactly(4).times
-        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue StandardError
       end
 
       it "refreshes metadata between retries" do
         @cluster_metadata.should_receive(:update).exactly(4).times
-        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue StandardError
       end
 
-      it "returns false" do
-        expect(@sp.send_messages([Message.new(:topic => "topic", :value => "value")])).to eq(false)
+      it "raises an exception" do
+        expect {
+          @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+        }.to raise_error
       end
     end
 
     context "no retries" do
       before(:each) do
-        @mbts.stub!(:all_sent?).and_return(false)
+        @mbts.stub!(:pending_messages?).and_return(true)
         @sp = SyncProducer.new("test_client", [], max_send_retries: 0)
       end
 
       it "does not call sleep" do
         Kernel.should_receive(:sleep).exactly(0).times
-        @sp.send_messages([Message.new(:topic => "topic", :value => "value")])
+        @sp.send_messages([Message.new(:topic => "topic", :value => "value")]) rescue Errors::UnableToFetchMetadata
       end
     end
 
     context "succeeds on first attempt" do
       before(:each) do
-        @mbts.stub!(:all_sent?).and_return(true)
+        @mbts.stub!(:pending_messages?).and_return(false)
         @sp = SyncProducer.new("test_client", [])
       end
 
@@ -119,7 +123,7 @@ describe SyncProducer do
 
     context "succeeds on second attempt" do
       before(:each) do
-        @mbts.stub!(:all_sent?).and_return(false, true)
+        @mbts.stub!(:pending_messages?).and_return(true, false)
         @sp = SyncProducer.new("test_client", [])
       end
 

metadata

@@ -1,69 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: poseidon
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5.pre1
 platform: ruby
 authors:
 - Bob Potter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-26 00:00:00.000000000 Z
+date: 2014-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.13.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.13.0
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: daemon_controller
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: A Kafka (http://kafka.apache.org/) producer and consumer
@@ -73,10 +59,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .travis.yml
-- .yardopts
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- ".yardopts"
 - CHANGES.md
 - Gemfile
 - LICENSE.txt
@@ -111,9 +97,11 @@ files:
 - lib/poseidon/sync_producer.rb
 - lib/poseidon/topic_metadata.rb
 - lib/poseidon/version.rb
+- log/.gitkeep
 - poseidon.gemspec
-- spec/bin/kafka-run-class.sh
 - spec/integration/multiple_brokers/consumer_spec.rb
+- spec/integration/multiple_brokers/metadata_failures_spec.rb
+- spec/integration/multiple_brokers/rebalance_spec.rb
 - spec/integration/multiple_brokers/round_robin_spec.rb
 - spec/integration/multiple_brokers/spec_helper.rb
 - spec/integration/simple/compression_spec.rb
@@ -153,23 +141,24 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Poseidon is a producer and consumer implementation for Kafka >= 0.8
 test_files:
-- spec/bin/kafka-run-class.sh
 - spec/integration/multiple_brokers/consumer_spec.rb
+- spec/integration/multiple_brokers/metadata_failures_spec.rb
+- spec/integration/multiple_brokers/rebalance_spec.rb
 - spec/integration/multiple_brokers/round_robin_spec.rb
 - spec/integration/multiple_brokers/spec_helper.rb
 - spec/integration/simple/compression_spec.rb

data/spec/bin/kafka-run-class.sh

@@ -1,65 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-# 
-#    http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ];
-then
-  echo "USAGE: $0 classname [opts]"
-  exit 1
-fi
-
-SCALA_VERSION=2.8.0
-
-# assume all dependencies have been packaged into one jar with sbt-assembly's task "assembly-package-dependency"
-for file in $KAFKA_PATH/core/target/scala-2.8.0/*.jar;
-do
-  CLASSPATH=$CLASSPATH:$file
-done
-
-for file in $KAFKA_PATH/perf/target/scala-${SCALA_VERSION}/kafka*.jar;
-do
-  CLASSPATH=$CLASSPATH:$file
-done
-
-# classpath addition for release
-for file in $KAFKA_PATH/libs/*.jar;
-do
-  CLASSPATH=$CLASSPATH:$file
-done
-
-for file in $KAFKA_PATH/kafka*.jar;
-do
-  CLASSPATH=$CLASSPATH:$file
-done
-
-if [ -z "$KAFKA_JMX_OPTS" ]; then
-  KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false "
-fi
-
-if [ -z "$KAFKA_OPTS" ]; then
-  KAFKA_OPTS="-Xmx512M -server  -Dlog4j.configuration=file:$KAFKA_PATH/config/log4j.properties"
-fi
-
-if [  $JMX_PORT ]; then
-  KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT "
-fi
-
-if [ -z "$JAVA_HOME" ]; then
-  JAVA="java"
-else
-  JAVA="$JAVA_HOME/bin/java"
-fi
-
-exec $JAVA $KAFKA_OPTS $KAFKA_JMX_OPTS -cp $CLASSPATH "$@"