poseidon 0.0.1 → 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 768653a82da925b840342126faf304153c1b1455
+  data.tar.gz: 82c3ed1cd5746d582666e7e04aec8cecca986b91
+SHA512:
+  metadata.gz: e67ed28694dcb0c8f1596ada7a61b6c53ddaf36c639eaf6cbd69fa95390daefdef901cf587057778aa6776ad769f9046bc66db34e1731ea1efff4f0d6f58a04f
+  data.tar.gz: 861af0b1788b241e59de63a87514d3af0a60d04f6d9b14dca22966a4172a5c45c19888bd0ff1da0aa9df575829d1879d9e1e3c3cf3aebff4a7243a02fc65707e

data/CHANGES.md

@@ -0,0 +1,8 @@
+# 0.0.2
+
+* Added ability to create a partitioner consumer for a topic+partition using topic metadata.
+* Added PartitionConsumer#offset to return offset of the last fetch
+
+# 0.0.1
+
+* Initial release

data/README.md

@@ -4,10 +4,13 @@ Poseidon is a Kafka client. Poseidon only supports the 0.8 API and above.
 
 **Until 1.0.0 this should be considered ALPHA software and not neccessarily production ready.**
 
-* [API Documentation](http://rubydoc.info/github/bpot/poseidon)
-
 ## Usage
 
+### API Documentation
+
+* [Latest release](http://rubydoc.info/gems/poseidon)
+* [Github master](http://rubydoc.info/github/bpot/poseidon)
+
 ### Installing a Kafka broker locally
 
 Follow the [instructions](https://cwiki.apache.org/KAFKA/kafka-08-quick-start.html) on the Kafka wiki to build Kafka 0.8 and get a test broker up and running.

data/lib/poseidon/cluster_metadata.rb

@@ -47,6 +47,16 @@ module Poseidon
       @brokers[broker_id]
     end
 
+    # Return lead broker for topic and partition
+    def lead_broker_for_partition(topic_name, partition)
+      broker_id = @topic_metadata[topic_name].partition_leader(partition)
+      if broker_id
+        @brokers[broker_id]
+      else
+        nil
+      end
+    end
+
     private
     def update_topics(topics)
       topics.each do |topic|

data/lib/poseidon/partition_consumer.rb

@@ -10,6 +10,25 @@ module Poseidon
     # as recent as the last fetch call.
     attr_reader :highwater_mark
 
+    attr_reader :host, :port
+
+    attr_reader :offset
+
+    # Returns a consumer pointing at the lead broker for the partition.
+    #
+    # Eventually this will be replaced by higher level consumer functionality,
+    # this is a stop-gap.
+    #
+    def self.consumer_for_partition(client_id, seed_brokers, topic, partition, offset, options = {})
+      broker_pool = BrokerPool.new(client_id, seed_brokers)
+
+      cluster_metadata = ClusterMetadata.new
+      cluster_metadata.update(broker_pool.fetch_metadata([topic]))
+
+      broker = cluster_metadata.lead_broker_for_partition(topic, partition)
+      new(client_id, broker.host, broker.port, topic, partition, offset, options)
+    end
+
     # Create a new consumer which reads the specified topic and partition from
     # the host.
     #
@@ -37,6 +56,9 @@ module Poseidon
     #
     # @api public
     def initialize(client_id, host, port, topic, partition, offset, options = {})
+      @host = host
+      @port = port
+
       @connection = Connection.new(host, port, client_id)
       @topic = topic
       @partition = partition

data/lib/poseidon/topic_metadata.rb

@@ -61,5 +61,14 @@ module Poseidon
     def available_partition_count
       @available_partition_count ||= available_partition_leader_ids.count
     end
+
+    def partition_leader(partition_id)
+      partition = struct.partitions.find { |p| p.id == partition_id }
+      if partition
+        partition.leader
+      else
+        nil
+      end
+    end
   end
 end

data/lib/poseidon/version.rb

@@ -1,4 +1,4 @@
 module Poseidon
   # Unstable! API May Change!
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/poseidon.gemspec

@@ -11,6 +11,7 @@ Gem::Specification.new do |gem|
   gem.description   = %q{A Kafka (http://kafka.apache.org/) producer and consumer}
   gem.summary       = %q{Poseidon is a producer and consumer implementation for Kafka >= 0.8}
   gem.homepage      = "https://github.com/bpot/poseidon"
+  gem.licenses      = ["MIT"]
 
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/spec/integration/multiple_brokers/consumer_spec.rb

@@ -0,0 +1,43 @@
+require 'integration/multiple_brokers/spec_helper'
+
+describe "consuming with multiple brokers" do
+  before(:each) do
+    # autocreate the topic by asking for information about it
+    c = Connection.new("localhost", 9092, "metadata_fetcher")
+    md = c.topic_metadata(["test"])
+    sleep 1
+  end
+
+  it "finds the lead broker for each partition" do
+    brokers = Set.new
+    0.upto(2) do |partition|
+      pc = PartitionConsumer.consumer_for_partition("test_client",
+                                                    ["localhost:9092"],
+                                                    "test", partition,
+                                                    :earliest_offset)
+
+      brokers.add("#{pc.host}:#{pc.port}")
+    end
+    expect(brokers.size).to eq(3)
+  end
+
+  it "consumes from all partitions" do
+    @p = Producer.new(["localhost:9092","localhost:9093","localhost:9094"], "test",
+                     :required_acks => 1)
+
+    msgs = 24.times.map { |n| "hello_#{n}" }
+    msgs.each do |msg|
+      @p.send_messages([MessageToSend.new("test", msg)])
+    end
+
+    fetched_messages = []
+    0.upto(2) do |partition|
+      pc = PartitionConsumer.consumer_for_partition("test_client",
+                                                    ["localhost:9092"],
+                                                    "test", partition,
+                                                    :earliest_offset)
+      fetched_messages.push(*pc.fetch)
+    end
+    expect(fetched_messages.map(&:value).sort).to eq(msgs.sort)
+  end
+end

data/spec/unit/cluster_metadata_spec.rb

@@ -5,8 +5,8 @@ describe ClusterMetadata do
   describe "populated" do
     before(:each) do
       partitions = [
-        PartitionMetadata.new(1, 1, [1,2], [1,2], nil),
-        PartitionMetadata.new(2, 2, [2,1], [2,1], nil)
+        PartitionMetadata.new(nil, 1, 1, [1,2], [1,2]),
+        PartitionMetadata.new(nil, 2, 2, [2,1], [2,1])
       ]
       topics = [TopicMetadata.new(TopicMetadataStruct.new(nil, "test", partitions))]
 
@@ -37,5 +37,10 @@ describe ClusterMetadata do
       broker = @cm.broker(1)
       expect(broker).to eq(@mr.brokers.first)
     end
+
+    it "provides the lead broker for a partition" do
+      expect(@cm.lead_broker_for_partition("test",1).id).to eq(1)
+      expect(@cm.lead_broker_for_partition("test",2).id).to eq(2)
+    end
   end
 end

data/spec/unit/topic_metadata_spec.rb

@@ -3,15 +3,23 @@ require 'spec_helper'
 describe TopicMetadata do
   context "encoding" do
     it "roundtrips" do
-      partition_metadata = PartitionMetadata.new(0, 0, 0, [0], [0])
+      partition_metadata = Protocol::PartitionMetadata.new(0, 0, 0, [0], [0])
       partitions = [partition_metadata]
-      tm = TopicMetadata.new(TopicMetadataStruct.new(0, "topic", partitions))
+      tm = TopicMetadata.new(Protocol::TopicMetadataStruct.new(0, "topic", partitions))
 
-      request_buffer = RequestBuffer.new
+      request_buffer = Protocol::RequestBuffer.new
       tm.write(request_buffer)
 
-      response_buffer = ResponseBuffer.new(request_buffer.to_s)
+      response_buffer = Protocol::ResponseBuffer.new(request_buffer.to_s)
       expect(TopicMetadata.read(response_buffer)).to eq(tm)
     end
   end
+
+  it 'determines leader for a partition' do
+    partition_metadata = Protocol::PartitionMetadata.new(0, 0, 0, [0], [0])
+    partitions = [partition_metadata]
+    tm = TopicMetadata.new(Protocol::TopicMetadataStruct.new(0, "topic", partitions))
+
+    expect(tm.partition_leader(0)).to eq(0)
+  end
 end

metadata

@@ -1,78 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: poseidon
 version: !ruby/object:Gem::Version
-  version: 0.0.1
-  prerelease: 
+  version: 0.0.2
 platform: ruby
 authors:
 - Bob Potter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-11 00:00:00.000000000 Z
+date: 2013-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: daemon_controller
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: A Kafka (http://kafka.apache.org/) producer and consumer
@@ -86,6 +77,7 @@ files:
 - .rspec
 - .travis.yml
 - .yardopts
+- CHANGES.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -121,6 +113,7 @@ files:
 - lib/poseidon/version.rb
 - poseidon.gemspec
 - spec/bin/kafka-run-class.sh
+- spec/integration/multiple_brokers/consumer_spec.rb
 - spec/integration/multiple_brokers/round_robin_spec.rb
 - spec/integration/multiple_brokers/spec_helper.rb
 - spec/integration/simple/compression_spec.rb
@@ -150,31 +143,32 @@ files:
 - spec/unit/sync_producer_spec.rb
 - spec/unit/topic_metadata_spec.rb
 homepage: https://github.com/bpot/poseidon
-licenses: []
+licenses:
+- MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Poseidon is a producer and consumer implementation for Kafka >= 0.8
 test_files:
 - spec/bin/kafka-run-class.sh
+- spec/integration/multiple_brokers/consumer_spec.rb
 - spec/integration/multiple_brokers/round_robin_spec.rb
 - spec/integration/multiple_brokers/spec_helper.rb
 - spec/integration/simple/compression_spec.rb