poseidon_cluster 0.1.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 54b28361ea6e2d74ad75503560e29230f928cf87
-  data.tar.gz: dce02dfed81efe6a59c210391d07c74cf3781083
+  metadata.gz: 70266dd2ea6c22a6e57d802e8a30ad722228b2ad
+  data.tar.gz: 75eaf7ea9e37bb9c74ef7c5291b3fb28aa074096
 SHA512:
-  metadata.gz: ac690a44445aa513d2131339f8a7bb06dfe0e442d569e1389e1aa6959c01b80b5224dee4c8733aff41d2629803a847335a4a794ae8adb59ca5bd2f6c4a0b15cc
-  data.tar.gz: 117990bd5fd6badf2bdc9286ecba6d1ec0eaaca0b87ebaaf6fb385fae371d4669adc2527593d177ffe28caecf210f4af4049873e5d0e3ef1480ac9069405b17d
+  metadata.gz: fbb88da9cdb10569defc933e51bcde385b17813408f57d443562b531a851f046b1868f34f59e9ccb682ba0c9223c58e9fbca998cbf166ad9832bfd69723cc127
+  data.tar.gz: 1f89ca21c4d5aac75c757069c26a7a0e8d1cec32993ccd9dd7402e35096b7c65f1450d7739d3ad94fd5e1792405eca7bc7e4ead246ef954dfec3f673e2afa678

data/.gitignore

@@ -5,3 +5,4 @@ doc/
 .bundle/
 pkg/
 coverage/
+*.log

data/.travis.yml

@@ -1,9 +1,7 @@
 language: ruby
 rvm:
-  - 2.1.0
   - 2.0.0
-  - 1.9.3
-  - jruby-19mode
-env:
-  - SLOW=1
-script: bundle exec rake spec:coveralls
+  - 2.1.5
+script: bundle exec rake spec:coveralls scenario
+matrix:
+  fast_finish: true

data/Gemfile

@@ -1,7 +1,6 @@
 source "https://rubygems.org"
 
 gemspec
-gem "poseidon", git: "https://github.com/dim/poseidon.git"
 
 platform :jruby do
   gem 'slyphon-log4j', '= 1.2.15'

data/Gemfile.lock

@@ -1,59 +1,76 @@
-GIT
-  remote: https://github.com/dim/poseidon.git
-  revision: f853e8f07a6f2ff4a520d3f7e1dac030453a18ed
-  specs:
-    poseidon (0.0.4)
-
 PATH
   remote: .
   specs:
-    poseidon_cluster (0.1.1)
-      poseidon
+    poseidon_cluster (0.3.0)
+      poseidon (>= 0.0.5, < 0.1.0)
       zk
 
 GEM
   remote: https://rubygems.org/
   specs:
-    coveralls (0.7.0)
-      multi_json (~> 1.3)
-      rest-client
-      simplecov (>= 0.7)
-      term-ansicolor
-      thor
+    coveralls (0.8.1)
+      json (~> 1.8)
+      rest-client (>= 1.6.8, < 2)
+      simplecov (~> 0.10.0)
+      term-ansicolor (~> 1.3)
+      thor (~> 0.19.1)
     diff-lcs (1.2.5)
-    docile (1.1.2)
+    docile (1.1.5)
+    domain_name (0.5.24)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    json (1.8.3)
+    json (1.8.3-java)
     little-plugger (1.1.3)
-    logging (1.7.2)
+    logging (1.8.2)
       little-plugger (>= 1.1.3)
-    mime-types (2.1)
-    multi_json (1.8.4)
-    rake (10.1.1)
-    rest-client (1.6.7)
-      mime-types (>= 1.16)
-    rspec (2.14.1)
-      rspec-core (~> 2.14.0)
-      rspec-expectations (~> 2.14.0)
-      rspec-mocks (~> 2.14.0)
-    rspec-core (2.14.7)
-    rspec-expectations (2.14.4)
-      diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.14.4)
-    simplecov (0.8.2)
+      multi_json (>= 1.8.4)
+    mime-types (2.6.1)
+    multi_json (1.11.0)
+    netrc (0.10.3)
+    poseidon (0.0.5)
+    rake (10.4.2)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.3)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-its (1.2.0)
+      rspec-core (>= 3.0.0)
+      rspec-expectations (>= 3.0.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
+    simplecov (0.10.0)
       docile (~> 1.1.0)
-      multi_json
-      simplecov-html (~> 0.8.0)
-    simplecov-html (0.8.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
     slyphon-log4j (1.2.15)
     slyphon-zookeeper_jar (3.3.5-java)
-    term-ansicolor (1.2.2)
-      tins (~> 0.8)
-    thor (0.18.1)
-    tins (0.13.1)
-    yard (0.8.7.3)
-    zk (1.9.3)
-      logging (~> 1.7.2)
+    term-ansicolor (1.3.0)
+      tins (~> 1.0)
+    thor (0.19.1)
+    tins (1.5.2)
+    unf (0.1.4)
+      unf_ext
+    unf (0.1.4-java)
+    unf_ext (0.0.7.1)
+    yard (0.8.7.6)
+    zk (1.9.5)
+      logging (~> 1.8.2)
       zookeeper (~> 1.4.0)
-    zookeeper (1.4.8)
+    zookeeper (1.4.10)
 
 PLATFORMS
   java
@@ -62,10 +79,10 @@ PLATFORMS
 DEPENDENCIES
   bundler
   coveralls
-  poseidon!
   poseidon_cluster!
   rake
   rspec
+  rspec-its
   slyphon-log4j (= 1.2.15)
   slyphon-zookeeper_jar (= 3.3.5)
   yard

data/README.md

@@ -1,6 +1,6 @@
 # Poseidon Cluster [![Build Status](https://travis-ci.org/bsm/poseidon_cluster.png?branch=master)](https://travis-ci.org/bsm/poseidon_cluster) [![Coverage Status](https://coveralls.io/repos/bsm/poseidon_cluster/badge.png?branch=master)](https://coveralls.io/r/bsm/poseidon_cluster?branch=master)
 
-Poseidon Cluster is a cluster extension the excellent [Poseidon](http://github.com/bpot/poseidon) Ruby client for Kafka 0.8+. It implements the distribution concept of self-rebalancing *Consumer Groups* and supports the consumption of a single topic from multiple instances.
+Poseidon Cluster is a cluster extension of the excellent [Poseidon](http://github.com/bpot/poseidon) Ruby client for Kafka 0.8+. It implements the distribution concept of self-rebalancing *Consumer Groups* and supports the consumption of a single topic from multiple instances.
 
 Consumer group instances share a common group name, and each message published to a topic is delivered to one instance within each subscribing consumer group. Consumer instances can be in separate processes or on separate machines.
 

data/Rakefile

@@ -13,4 +13,9 @@ namespace :spec do
   task coveralls: [:spec, 'coveralls:push']
 end
 
+desc "Run full integration test scenario"
+task :scenario do
+  load File.expand_path("../scenario/run.rb", __FILE__)
+end
+
 task default: :spec

data/lib/poseidon/cluster.rb

@@ -15,6 +15,12 @@ module Poseidon::Cluster
     @@sem.synchronize { @@inc += 1; @@inc = 1 if @@inc > MAX_INT32; @@inc }
   end
 
+  # @return [String] an globally unique identifier
+  # @api private
+  def self.guid
+    [::Socket.gethostname, ::Process.pid, ::Time.now.to_i, inc!].join("-")
+  end
+
 end
 
 %w|consumer_group|.each do |name|

data/lib/poseidon/consumer_group.rb

@@ -21,7 +21,7 @@
 #
 # @api public
 class Poseidon::ConsumerGroup
-  DEFAULT_CLAIM_TIMEOUT = 10
+  DEFAULT_CLAIM_TIMEOUT = 30
   DEFAULT_LOOP_DELAY = 1
 
   # Poseidon::ConsumerGroup::Consumer is internally used by Poseidon::ConsumerGroup.
@@ -37,6 +37,8 @@ class Poseidon::ConsumerGroup
     def initialize(group, partition, options = {})
       broker = group.leader(partition)
       offset = group.offset(partition)
+      offset = (options[:trail] ? :latest_offset : :earliest_offset) if offset == 0
+      options.delete(:trail)
       super group.id, broker.host, broker.port, group.topic, partition, offset, options
     end
 
@@ -87,16 +89,21 @@ class Poseidon::ConsumerGroup
   # @option options [Integer] :min_bytes Smallest amount of data the server should send us. Default: 0 (Send us data as soon as it is ready)
   # @option options [Integer] :claim_timeout Maximum number of seconds to wait for a partition claim. Default: 10
   # @option options [Integer] :loop_delay Number of seconds to delay the next fetch (in #fetch_loop) if nothing was returned. Default: 1
+  # @option options [Integer] :socket_timeout_ms broker connection wait timeout in ms. Default: 10000
   # @option options [Boolean] :register Automatically register instance and start consuming. Default: true
+  # @option options [Boolean] :trail Starts reading messages from the latest partitions offsets and skips 'old' messages . Default: false
   #
   # @api public
   def initialize(name, brokers, zookeepers, topic, options = {})
     @name       = name
     @topic      = topic
     @zk         = ::ZK.new(zookeepers.join(","))
+    # Poseidon::BrokerPool doesn't provide default value for this option
+    # Configuring default value like this isn't beautiful, though.. by kssminus
+    options[:socket_timeout_ms] ||= 10000
     @options    = options
     @consumers  = []
-    @pool       = ::Poseidon::BrokerPool.new(id, brokers)
+    @pool       = ::Poseidon::BrokerPool.new(id, brokers, options[:socket_timeout_ms])
     @mutex      = Mutex.new
     @registered = false
 
@@ -105,7 +112,7 @@ class Poseidon::ConsumerGroup
 
   # @return [String] a globally unique identifier
   def id
-    @id ||= [name, ::Socket.gethostname, ::Process.pid, ::Time.now.to_i, ::Poseidon::Cluster.inc!].join("-")
+    @id ||= [name, Poseidon::Cluster.guid].join("-")
   end
 
   # @return [Hash<Symbol,String>] registry paths
@@ -158,7 +165,7 @@ class Poseidon::ConsumerGroup
   # Closes the consumer group gracefully, only really useful in tests
   # @api private
   def close
-    release_all!
+    @mutex.synchronize { release_all! }
     zk.close
   end
 
@@ -189,7 +196,7 @@ class Poseidon::ConsumerGroup
   def partitions
     return [] unless topic_metadata
 
-    topic_metadata.partitions.sort_by do |part|
+    topic_metadata.available_partitions.sort_by do |part|
       broker = metadata.brokers[part.leader]
       [broker.host, broker.port].join(":")
     end
@@ -220,15 +227,16 @@ class Poseidon::ConsumerGroup
   #
   # @api public
   def checkout(opts = {})
-    return false if @rebalancing
+    consumer = nil
+    commit   = @mutex.synchronize do
+      consumer = @consumers.shift
+      return false unless consumer
 
-    consumer = @consumers.shift
-    return false unless consumer
-
-    @consumers.push(consumer)
-    result = yield(consumer)
+      @consumers.push consumer
+      yield consumer
+    end
 
-    unless opts[:commit] == false || result == false
+    unless opts[:commit] == false || commit == false
       commit consumer.partition, consumer.offset
     end
     true
@@ -358,7 +366,29 @@ class Poseidon::ConsumerGroup
     # * let POS be our index position in CG and let N = size(PT)/size(CG)
     # * assign partitions from POS*N to (POS+1)*N-1
     def rebalance!
-      @mutex.synchronize { perform_rebalance! }
+      return if @pending
+
+      @pending = true
+      @mutex.synchronize do
+        @pending = nil
+
+        release_all!
+        reload
+
+        ids = zk.children(registries[:consumer], watch: true)
+        pms = partitions
+        rng = self.class.pick(pms.size, ids, id)
+
+        pms[rng].each do |pm|
+          if @pending
+            release_all!
+            break
+          end
+
+          consumer = claim!(pm.id)
+          @consumers.push(consumer) if consumer
+        end if rng
+      end
     end
 
     # Release all consumer claims
@@ -367,12 +397,17 @@ class Poseidon::ConsumerGroup
       @consumers.clear
     end
 
+  private
+
     # Claim the ownership of the partition for this consumer
     # @raise [Timeout::Error]
     def claim!(partition)
       path = claim_path(partition)
       Timeout.timeout options[:claim_timout] || DEFAULT_CLAIM_TIMEOUT do
-        sleep(0.1) while zk.create(path, id, ephemeral: true, ignore: :node_exists).nil?
+        while zk.create(path, id, ephemeral: true, ignore: :node_exists).nil?
+          return if @pending
+          sleep(0.1)
+        end
       end
       Consumer.new self, partition, options.dup
     end
@@ -382,8 +417,6 @@ class Poseidon::ConsumerGroup
       zk.delete claim_path(partition), ignore: :no_node
     end
 
-  private
-
     # @return [String] zookeeper ownership claim path
     def claim_path(partition)
       "#{registries[:owner]}/#{partition}"
@@ -399,21 +432,4 @@ class Poseidon::ConsumerGroup
       "#{registries[:consumer]}/#{id}"
     end
 
-    def perform_rebalance!
-      @rebalancing = true
-      reload
-      release_all!
-
-      ids = zk.children(registries[:consumer], watch: true)
-      pms = partitions
-      rng = self.class.pick(pms.size, ids, id)
-
-      pms[rng].each do |pm|
-        consumer = claim!(pm.id)
-        @consumers.push(consumer)
-      end if rng
-    ensure
-      @rebalancing = nil
-    end
-
 end

data/poseidon_cluster.gemspec

@@ -1,11 +1,11 @@
 Gem::Specification.new do |s|
-  s.required_ruby_version = '>= 1.9.1'
+  s.required_ruby_version = '>= 2.0.0'
   s.required_rubygems_version = ">= 1.8.0"
 
   s.name        = File.basename(__FILE__, '.gemspec')
   s.summary     = "Poseidon cluster extensions"
   s.description = "Cluster extensions for Poseidon, a producer and consumer implementation for Kafka >= 0.8"
-  s.version     = "0.1.1"
+  s.version     = "0.3.0"
 
   s.authors     = ["Black Square Media"]
   s.email       = "info@blacksquaremedia.com"
@@ -13,14 +13,15 @@ Gem::Specification.new do |s|
 
   s.require_path = 'lib'
   s.files        = `git ls-files`.split("\n")
-  s.test_files   = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.test_files   = `git ls-files -- {test,spec,features,scenario}/*`.split("\n")
 
-  s.add_dependency "poseidon"
+  s.add_dependency "poseidon", ">= 0.0.5", "<0.1.0"
   s.add_dependency "zk"
 
   s.add_development_dependency "rake"
   s.add_development_dependency "bundler"
   s.add_development_dependency "rspec"
+  s.add_development_dependency "rspec-its"
   s.add_development_dependency "yard"
   s.add_development_dependency "coveralls"
 

data/scenario/.gitignore

@@ -0,0 +1 @@
+output.txt

data/scenario/consumer.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require 'bundler/setup'
+require 'poseidon_cluster'
+
+name     = ARGV[0].to_s
+output   = File.open(ARGV[1], "a")
+output.sync = true
+
+total    = 0
+consumer = Poseidon::ConsumerGroup.new "my-group", ["localhost:29092"], ["localhost:22181"], "my-topic", max_bytes: 256*1024
+consumer.fetch_loop do |n, messages|
+  break if name[0] > 'Q' && total > 0
+  messages.each do |m|
+    output.write "#{name},#{n},#{m.value}\n"
+  end
+  total += messages.size
+end

data/scenario/producer.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+require 'bundler/setup'
+require 'poseidon'
+
+limit, offset = ARGV[0].to_i, ARGV[1].to_i
+producer = Poseidon::Producer.new ["localhost:29092"], "poseidon-producer"
+
+while limit > 0 do
+  batch  = limit > 10000 ? 10000 : limit
+  limit -= batch
+
+  messages = (0...batch).map do
+    num = offset.to_s.rjust(8, "0")
+    offset += 1
+    Poseidon::MessageToSend.new "my-topic", num, Time.now.to_s+num
+  end
+
+  10.times do
+    ok = producer.send_messages messages
+    break if ok
+    sleep(1)
+  end
+end

data/scenario/run.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'timeout'
+require File.expand_path("../scenario", __FILE__)
+
+# Start Zookeeper & Kafka
+Scenario.run do
+  5.times do
+    produce 1000
+  end
+  consume "A"
+  consume "B"
+  consume "C"
+  checkpoint!
+
+  15.times { produce 1000 }
+  consume "D"
+  10.times { produce 1000 }
+  consume "X"
+  10.times { produce 1000 }
+  checkpoint!
+
+  20.times { produce 1000 }
+  consume "E"
+  consume "F"
+  15.times { produce 1000 }
+  consume "Y"
+  50.times { produce 100 }
+  20.times { produce 1000 }
+
+  checkpoint!
+end
+
+

data/scenario/scenario.rb

@@ -0,0 +1,134 @@
+require 'fileutils'
+require 'pathname'
+
+module Scenario
+  extend self
+
+  ROOT    = Pathname.new(File.expand_path("../", __FILE__))
+  VERSION = "0.8.1.1"
+  SERVER  = ROOT.join "kafka_2.10-#{VERSION}"
+
+  TOPIC_NAME = "my-topic"
+  KAFKA_BIN  = SERVER.join("bin", "kafka-server-start.sh")
+  KAFKA_CFG  = SERVER.join("config", "server-poseidon.properties")
+  KAFKA_TMP  = "/tmp/kafka-logs-poseidon"
+  ZOOKP_BIN  = SERVER.join("bin", "zookeeper-server-start.sh")
+  ZOOKP_CFG  = SERVER.join("config", "zookeeper-poseidon.properties")
+  ZOOKP_TMP  = "/tmp/zookeeper-poseidon"
+  LOG4J_CFG  = SERVER.join("config", "log4j.properties")
+  OUTPUT     = Scenario::ROOT.join("output.txt")
+
+  @@pids     = {}
+  @@total    = 0
+
+  def run(&block)
+    setup
+    instance_eval(&block)
+  rescue => e
+    abort [e, *e.backtrace[0,20]].join("\n")
+  ensure
+    teardown
+  end
+
+  def setup
+    FileUtils.rm_rf OUTPUT.to_s
+    configure
+
+    # Ensure all required files are present
+    [KAFKA_BIN, ZOOKP_BIN, KAFKA_CFG, ZOOKP_CFG].each do |path|
+      abort "Unable to locate #{path}. File does not exist!" unless path.file?
+    end
+
+    Signal.trap("INT") { teardown }
+
+    spawn KAFKA_BIN, KAFKA_CFG
+    spawn ZOOKP_BIN, ZOOKP_CFG
+    sleep(2)
+  end
+
+  def teardown
+    @@pids.each do |_, pid|
+      Process.kill :TERM, pid
+    end
+    sleep(1)
+    FileUtils.rm_rf KAFKA_TMP.to_s
+    FileUtils.rm_rf ZOOKP_TMP.to_s
+
+    fail! unless numlines == @@total
+  end
+
+  def configure
+    download
+
+    KAFKA_CFG.open("w") do |f|
+      f.write SERVER.join("config", "server.properties").read.
+        sub("=9092", "=29092").
+        sub(":2181", ":22181").
+        sub("num.partitions=2", "num.partitions=12").
+        sub("log.flush.interval.ms=1000", "log.flush.interval.ms=10").
+        sub("/tmp/kafka-logs", KAFKA_TMP)
+    end
+    ZOOKP_CFG.open("w") do |f|
+      f.write SERVER.join("config", "zookeeper.properties").read.
+        sub("/tmp/zookeeper", ZOOKP_TMP).
+        sub("=2181", "=22181")
+    end
+    content = LOG4J_CFG.read
+    LOG4J_CFG.open("w") do |f|
+      f.write content.gsub("INFO", "FATAL")
+    end if content.include?("INFO")
+  end
+
+  def download
+    return if SERVER.directory?
+    sh "cd #{ROOT} && curl http://www.mirrorservice.org/sites/ftp.apache.org/kafka/#{VERSION}/kafka_2.10-#{VERSION}.tgz | tar xz"
+  end
+
+  def checkpoint!(timeout = 10)
+    puts "--> Verifying #{@@total}"
+    timeout.times do
+      if numlines > @@total
+        break
+      elsif numlines < @@total
+        sleep(1)
+      else
+        return
+      end
+    end
+    fail!
+  end
+
+  def consume(name)
+    puts "--> Launching consumer #{name}"
+    spawn ROOT.join("consumer.rb"), name, OUTPUT
+  end
+
+  def produce(count)
+    puts "--> Producing messages #{@@total}-#{@@total+count-1}"
+    sh ROOT.join("producer.rb"), count, @@total
+    @@total += count
+  end
+
+  def numlines
+    `wc -l #{OUTPUT} 2> /dev/null`.to_i
+  end
+
+  def abort(message)
+    Kernel.abort "ERROR: #{message}"
+  end
+
+  def fail!
+    Kernel.abort "FAILED: expected #{@@total} but was #{numlines}"
+  end
+
+  def sh(*bits)
+    cmd = bits.join(" ")
+    system(cmd) || abort(cmd)
+  end
+
+  def spawn(*args)
+    cmd = args.join(" ")
+    @@pids[cmd] = Process.spawn(cmd)
+  end
+
+end

data/spec/lib/poseidon/cluster_spec.rb

@@ -12,4 +12,8 @@ describe Poseidon::Cluster do
     (described_class.inc! - num).should == 502
   end
 
+  it 'should generate GUIDs' do
+    described_class.guid.should match(/\A[\w\-\.]+?\-\d{1,5}\-\d{10}\-\d{1,3}\z/)
+  end
+
 end

data/spec/lib/poseidon/consumer_group_spec.rb

@@ -2,24 +2,14 @@ require 'spec_helper'
 
 describe Poseidon::ConsumerGroup do
 
-  def new_group
-    group = described_class.new "my-group", ["localhost:29092", "localhost:29091"], ["localhost:22181"], TOPIC_NAME
-    groups.push(group)
-    group
-  end
-
   def fetch_response(n)
     set = Poseidon::MessageSet.new
     n.times {|i| set << Poseidon::Message.new(value: "value", key: "key", offset: i) }
     pfr = Poseidon::Protocol::PartitionFetchResponse.new(0, 0, 100, set)
-    tfr = Poseidon::Protocol::TopicFetchResponse.new(TOPIC_NAME, [pfr])
+    tfr = Poseidon::Protocol::TopicFetchResponse.new("mytopic", [pfr])
     Poseidon::Protocol::FetchResponse.new(nil, [tfr])
   end
 
-  let :groups do
-    []
-  end
-
   let :brokers do
     [ Poseidon::Protocol::Broker.new(1, "localhost", 29092),   # id,host,port
       Poseidon::Protocol::Broker.new(2, "localhost", 29091), ]
@@ -31,53 +21,59 @@ describe Poseidon::ConsumerGroup do
   end
 
   let :topics do
-    [ Poseidon::TopicMetadata.new(Poseidon::Protocol::TopicMetadataStruct.new(0, TOPIC_NAME, partitions)) ]
+    [ Poseidon::TopicMetadata.new(Poseidon::Protocol::TopicMetadataStruct.new(0, "mytopic", partitions)) ]
   end
 
   let :metadata do
     Poseidon::Protocol::MetadataResponse.new nil, brokers.dup, topics.dup
   end
 
-  subject { new_group }
-  before do
-    Poseidon::ConsumerGroup.any_instance.stub(:sleep)
-    Poseidon::BrokerPool.any_instance.stub(:fetch_metadata_from_broker).and_return(metadata)
-    Poseidon::Connection.any_instance.stub(:fetch).with{|_, _, req| req[0].partition_fetches[0].partition == 0 }.and_return(fetch_response(10))
-    Poseidon::Connection.any_instance.stub(:fetch).with{|_, _, req| req[0].partition_fetches[0].partition == 1 }.and_return(fetch_response(5))
+  let :zk_client do
+    double "ZK", mkdir_p: nil, get: nil, set: nil, delete: nil, create: "/path", register: nil, children: ["my-group-UNIQUEID"], close: nil
   end
-  after do
-    subject.zk.rm_rf "/consumers/#{subject.name}"
-    groups.each(&:close)
+
+  let(:group) { described_class.new "my-group", ["localhost:29092", "localhost:29091"], ["localhost:22181"], "mytopic" }
+  subject     { group }
+
+  before do
+    allow(ZK).to receive_messages(new: zk_client)
+    allow(Poseidon::Cluster).to receive_messages(guid: "UNIQUEID")
+    allow_any_instance_of(Poseidon::ConsumerGroup).to receive(:sleep)
+    allow_any_instance_of(Poseidon::PartitionConsumer).to receive_messages(resolve_offset_if_necessary: 0)
+    allow_any_instance_of(Poseidon::BrokerPool).to receive_messages(fetch_metadata_from_broker: metadata)
+
+    allow_any_instance_of(Poseidon::Connection).to receive(:fetch).with(10000, 1, ->req { req[0].partition_fetches[0].partition == 0 }).and_return(fetch_response(10))
+    allow_any_instance_of(Poseidon::Connection).to receive(:fetch).with(10000, 1, ->req { req[0].partition_fetches[0].partition == 1 }).and_return(fetch_response(5))
   end
 
   it               { should be_registered }
   its(:name)       { should == "my-group" }
-  its(:topic)      { should == TOPIC_NAME }
+  its(:topic)      { should == "mytopic" }
   its(:pool)       { should be_instance_of(Poseidon::BrokerPool) }
-  its(:id)         { should match(/\Amy-group\-[\w\-\.]+?\-\d{1,5}\-\d{10}\-\d{1,3}\z/) }
-  its(:zk)         { should be_instance_of(ZK::Client::Threaded) }
+  its(:id)         { should == "my-group-UNIQUEID" }
+  its(:zk)         { should be(zk_client) }
 
   its(:claimed)        { should == [0, 1] }
   its(:metadata)       { should be_instance_of(Poseidon::ClusterMetadata) }
   its(:topic_metadata) { should be_instance_of(Poseidon::TopicMetadata) }
   its(:registries)     { should == {
     consumer: "/consumers/my-group/ids",
-    owner:    "/consumers/my-group/owners/my-topic",
-    offset:   "/consumers/my-group/offsets/my-topic",
+    owner:    "/consumers/my-group/owners/mytopic",
+    offset:   "/consumers/my-group/offsets/mytopic",
   }}
 
   its("metadata.brokers.keys") { should =~ [1,2] }
   its("topic_metadata.partition_count") { should == 2 }
 
-  it "should register with zookeeper" do
-    subject.zk.children("/consumers/my-group/ids").should include(subject.id)
-    stat = subject.zk.stat("/consumers/my-group/ids")
-    stat.ephemeral_owner.should be(0)
+  it "should register with zookeeper and rebalance" do
+    zk_client.should_receive(:mkdir_p).with("/consumers/my-group/ids")
+    zk_client.should_receive(:mkdir_p).with("/consumers/my-group/owners/mytopic")
+    zk_client.should_receive(:mkdir_p).with("/consumers/my-group/offsets/mytopic")
+    zk_client.should_receive(:create).with("/consumers/my-group/ids/my-group-UNIQUEID", "{}", ephemeral: true)
+    zk_client.should_receive(:register).with("/consumers/my-group/ids")
+    described_class.any_instance.should_receive :rebalance!
 
-    data, stat = subject.zk.get("/consumers/my-group/ids/#{subject.id}")
-    data.should == "{}"
-    stat.num_children.should == 0
-    stat.ephemeral_owner.should > 0
+    subject
   end
 
   it "should sort partitions by leader address" do
@@ -93,12 +89,13 @@ describe Poseidon::ConsumerGroup do
   end
 
   it "should return the offset for each partition" do
+    zk_client.should_receive(:get).with("/consumers/my-group/offsets/mytopic/0", ignore: :no_node).and_return([nil])
     subject.offset(0).should == 0
-    subject.offset(1).should == 0
-    subject.offset(2).should == 0
-    subject.fetch {|*| true }
-    subject.offset(0).should == 0
-    subject.offset(1).should == 5
+
+    zk_client.should_receive(:get).with("/consumers/my-group/offsets/mytopic/1", ignore: :no_node).and_return(["21", nil])
+    subject.offset(1).should == 21
+
+    zk_client.should_receive(:get).with("/consumers/my-group/offsets/mytopic/2", ignore: :no_node).and_return(["0", nil])
     subject.offset(2).should == 0
   end
 
@@ -109,8 +106,8 @@ describe Poseidon::ConsumerGroup do
   end
 
   it "should checkout individual partition consumers (atomically)" do
-    subject.checkout {|c| c.partition.should == 1 }.should be_true
-    subject.checkout {|c| c.partition.should == 0 }.should be_true
+    subject.checkout {|c| c.partition.should == 1 }.should be_truthy
+    subject.checkout {|c| c.partition.should == 0 }.should be_truthy
 
     n = 0
     a = Thread.new do
@@ -127,89 +124,86 @@ describe Poseidon::ConsumerGroup do
     n.should == 400
   end
 
+  describe "consumer" do
+    subject { described_class::Consumer.new group, 1 }
+    before  { group.stub(:offset).with(1).and_return(432) }
+
+    it { should be_a(Poseidon::PartitionConsumer) }
+    its(:offset) { should == 432 }
+
+    it 'should start with the earliest offset if none stored' do
+      group.unstub(:offset)
+      subject.offset.should == :earliest_offset
+    end
+
+    it 'should start with the latest offset if none stored and in trailing mode' do
+      group.unstub(:offset)
+      trailing_consumer = described_class::Consumer.new group, 1, {trail: true}
+      trailing_consumer.offset.should == :latest_offset
+    end
+
+  end
+
   describe "rebalance" do
 
     it "should watch out for new consumers joining/leaving" do
-      subject.should_receive(:rebalance!).twice.and_call_original
-      new_group.should_receive(:rebalance!).once.and_call_original
-      new_group
+      described_class.any_instance.should_receive(:rebalance!)
+      subject
     end
 
     it "should distribute available partitions between consumers" do
       subject.claimed.should == [0, 1]
-
-      b = new_group
-      wait_for { subject.claimed.size > 0 }
-      wait_for { b.claimed.size > 0 }
-      subject.claimed.should == [1]
-      b.claimed.should == [0]
-
-      c = new_group
-      b.close
-      wait_for { b.claimed.size < 0 }
-      wait_for { c.claimed.size > 0 }
-
-      subject.claimed.should == [1]
-      b.claimed.should == []
-      c.claimed.should == [0]
+      zk_client.stub children: ["my-group-UNIQUEID", "my-group-OTHERID"]
+      -> { subject.send :rebalance! }.should change { subject.claimed }.to([0])
+      zk_client.stub children: ["my-group-UNIQUEID", "my-group-OTHERID", "my-group-THIRDID"]
+      -> { subject.send :rebalance! }.should change { subject.claimed }.to([])
     end
 
     it "should allocate partitions correctly" do
       subject.claimed.should == [0, 1]
 
-      b = new_group
-      wait_for { subject.claimed.size > 0 }
-      wait_for { b.claimed.size > 0 }
-      subject.claimed.should == [1]
-      b.claimed.should == [0]
-
-      c = new_group
-      b.close
-      wait_for { b.claimed.size < 0 }
-      wait_for { c.claimed.size > 0 }
+      zk_client.stub children: ["my-group-UNIQUEID", "my-group-ZID"]
+      zk_client.should_receive(:delete).with("/consumers/my-group/owners/mytopic/1", ignore: :no_node)
+      -> { subject.send :rebalance! }.should change { subject.claimed }.to([1])
 
-      subject.claimed.should == [1]
-      b.claimed.should == []
-      c.claimed.should == [0]
+      zk_client.stub children: ["my-group-UNIQUEID", "my-group-ZID", "my-group-AID"]
+      -> { subject.send :rebalance! }.should change { subject.claimed }.to([0])
     end
 
   end
 
   describe "fetch" do
 
-    it "should return messages from owned partitions" do
+    it "should return messages from claimed partitions" do
       subject.fetch do |n, msg|
         n.should == 1
         msg.size.should == 5
-      end.should be_true
+      end.should be_truthy
 
       subject.fetch do |n, msg|
         n.should == 0
         msg.size.should == 10
-      end.should be_true
+      end.should be_truthy
 
       subject.fetch do |n, msg|
         n.should == 1
         msg.size.should == 5
-      end.should be_true
+      end.should be_truthy
     end
 
     it "should auto-commit fetched offset" do
-      -> {
-        subject.fetch {|n, _| n.should == 1 }
-      }.should change { subject.offset(1) }.from(0).to(5)
+      zk_client.should_receive(:set).with("/consumers/my-group/offsets/mytopic/1", "5")
+      subject.fetch {|n, _| n.should == 1 }
     end
 
     it "should skip auto-commits if requested" do
-      -> {
-        subject.fetch(commit: false) {|n, _| n.should == 1 }
-      }.should_not change { subject.offset(1) }
+      zk_client.should_not_receive(:set)
+      subject.fetch(commit: false) {|n, _| n.should == 1 }
     end
 
     it "should skip auto-commits if block results in false" do
-      -> {
-        subject.fetch {|n, _| n.should == 1; false }
-      }.should_not change { subject.offset(1) }
+      zk_client.should_not_receive(:set)
+      subject.fetch {|n, _| n.should == 1; false }
     end
 
     it "should return false when trying to fetch messages without a claim" do
@@ -217,12 +211,12 @@ describe Poseidon::ConsumerGroup do
       Poseidon::BrokerPool.any_instance.stub fetch_metadata_from_broker: no_topics
 
       subject.claimed.should == []
-      subject.fetch {|*|  }.should be_false
+      subject.fetch {|*|  }.should be_falsey
     end
 
     it "should return true even when no messages were fetched" do
       Poseidon::Connection.any_instance.stub fetch: fetch_response(0)
-      subject.fetch {|*|  }.should be_true
+      subject.fetch {|*|  }.should be_truthy
     end
 
   end

data/spec/spec_helper.rb

@@ -1,64 +1,14 @@
 require 'poseidon_cluster'
 require 'rspec'
-require 'fileutils'
-require 'pathname'
+require 'rspec/its'
 require 'coveralls'
 Coveralls.wear_merged!
 
-TOPIC_NAME  = "my-topic"
-KAFKA_LOCAL = File.expand_path("../kafka_2.8.0-0.8.0", __FILE__)
-KAFKA_ROOT  = Pathname.new(ENV["KAFKA_ROOT"] || KAFKA_LOCAL)
-
-module Poseidon::SpecHelper
-
-  def wait_for(&truth)
-    100.times do
-      break if truth.call
-      sleep(0.01)
-    end
-  end
-
-end
-
 RSpec.configure do |c|
-  c.include Poseidon::SpecHelper
-  c.filter_run_excluding slow: true unless ENV["SLOW"] == "1"
-  c.filter_run_excluding java: false if RUBY_PLATFORM == "java"
-
-  c.before :suite do
-    kafka_bin = KAFKA_ROOT.join("bin", "kafka-server-start.sh")
-    kafka_cfg = KAFKA_ROOT.join("config", "server-poseidon.properties")
-    zookp_bin = KAFKA_ROOT.join("bin", "zookeeper-server-start.sh")
-    zookp_cfg = KAFKA_ROOT.join("config", "zookeeper-poseidon.properties")
-
-    if KAFKA_ROOT.to_s == KAFKA_LOCAL && !kafka_bin.file?
-      puts "---> Downloading Kafka"
-      target = Pathname.new(File.expand_path("../", __FILE__))
-      system("cd #{target} && curl http://www.us.apache.org/dist/kafka/0.8.0/kafka_2.8.0-0.8.0.tar.gz | tar xz") ||
-        raise("Unable to download Kafka")
-
-      kafka_cfg.open("w") do |f|
-        f.write KAFKA_ROOT.join("config", "server.properties").read.sub("=9092", "=29092").sub(":2181", ":22181").sub("/tmp/kafka-logs", "/tmp/kafka-logs-poseidon")
-      end
-      zookp_cfg.open("w") do |f|
-        f.write KAFKA_ROOT.join("config", "zookeeper.properties").read.sub("=2181", "=22181")
-      end
-    end
-
-    # Ensure all required files are present
-    [kafka_bin, zookp_bin, kafka_cfg, zookp_cfg].each do |path|
-      raise "Unable to locate #{path}. File does not exist!" unless path.file?
-    end
-
-    # Start Zookeeper & Kafka
-    $ZOOKP_PID = spawn zookp_bin.to_s, zookp_cfg.to_s, out: '/dev/null'
-    $KAFKA_PID = spawn kafka_bin.to_s, kafka_cfg.to_s, out: '/dev/null'
+  c.expect_with :rspec do |c|
+    c.syntax = [:expect, :should]
   end
-
-  c.after :suite do
-    Process.kill :TERM, $KAFKA_PID if $KAFKA_PID
-    Process.kill :TERM, $ZOOKP_PID if $ZOOKP_PID
-    FileUtils.rm_rf "/tmp/kafka-logs-poseidon"
+  c.mock_with :rspec do |c|
+    c.syntax = [:expect, :should]
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: poseidon_cluster
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Black Square Media
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-05 00:00:00.000000000 Z
+date: 2015-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poseidon
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: zk
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +86,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-its
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -127,7 +147,11 @@ files:
 - lib/poseidon/consumer_group.rb
 - lib/poseidon_cluster.rb
 - poseidon_cluster.gemspec
-- spec/integration/poseidon/consumer_group_spec.rb
+- scenario/.gitignore
+- scenario/consumer.rb
+- scenario/producer.rb
+- scenario/run.rb
+- scenario/scenario.rb
 - spec/lib/poseidon/cluster_spec.rb
 - spec/lib/poseidon/consumer_group_spec.rb
 - spec/spec_helper.rb
@@ -142,7 +166,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.1
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -150,7 +174,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.8.0
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0.rc.1
+rubygems_version: 2.4.7
 signing_key: 
 specification_version: 4
 summary: Poseidon cluster extensions

data/spec/integration/poseidon/consumer_group_spec.rb

@@ -1,168 +0,0 @@
-require 'spec_helper'
-
-describe Poseidon::ConsumerGroup, integration: true do
-
-  def new_group(max_bytes = 1024*8, name = TOPIC_NAME)
-    described_class.new "my-group", ["localhost:29092"], ["localhost:22181"], name, max_bytes: max_bytes
-  end
-
-  def stored_offsets
-    { 0 => subject.offset(0), 1 => subject.offset(1) }
-  end
-
-  subject { new_group }
-  let(:consumed)  { Hash.new(0) }
-  let(:zookeeper) { ::ZK.new("localhost:22181") }
-
-  before :all do
-    producer = Poseidon::Producer.new(["localhost:29092"], "my-producer")
-    payload  = "data" * 10
-    messages = ("aa".."zz").map do |key|
-      Poseidon::MessageToSend.new(TOPIC_NAME, [key, payload].join(":"), key)
-    end
-
-    ok = false
-    100.times do
-      break if (ok = producer.send_messages(messages))
-      sleep(0.1)
-    end
-    pending "Unable to start Kafka instance." unless ok
-  end
-
-  after do
-    zookeeper.rm_rf "/consumers/my-group"
-  end
-
-  describe "small batches" do
-
-    it "should consume messages from all partitions" do
-      5.times do
-        subject.fetch {|n, msgs| consumed[n] += msgs.size }
-      end
-      consumed.values.inject(0, :+).should < 676
-
-      5.times do
-        subject.fetch {|n, msgs| consumed[n] += msgs.size }
-      end
-      consumed.keys.should =~ [0, 1]
-      consumed.values.inject(0, :+).should == 676
-      consumed.should == stored_offsets
-    end
-
-  end
-
-  describe "large batches" do
-    subject { new_group 1024 * 1024 * 10 }
-
-    it "should consume messages from all partitions" do
-      5.times do
-        subject.fetch {|n, msgs| consumed[n] += msgs.size }
-      end
-      consumed.keys.should =~ [0, 1]
-      consumed.values.inject(0, :+).should == 676
-      consumed.should == stored_offsets
-    end
-  end
-
-  describe "multi-thread fuzzing", slow: true do
-
-    def in_thread(batch_size, target, qu)
-      Thread.new do
-        sum   = 0
-        group = new_group(batch_size)
-        group.fetch_loop do |n, m|
-          break if sum > target || qu.size >= 676
-          sum += m.size
-          m.size.times { qu << true }
-        end
-        group.close
-        sum
-      end
-    end
-
-    it "should consume from multiple sources" do
-      q = Queue.new
-      a = in_thread(4001, 200, q)
-      b = in_thread(4002,  40, q)
-      c = in_thread(4003, 120, q)
-      d = in_thread(4004,  50, q)
-      e = in_thread(4005, 400, q)
-      vals = [a, b, c, d, e].map(&:value)
-      vals.inject(0, :+).should == 676
-
-      o1, _ = zookeeper.get "/consumers/my-group/offsets/#{TOPIC_NAME}/0"
-      o2, _ = zookeeper.get "/consumers/my-group/offsets/#{TOPIC_NAME}/1"
-      (o1.to_i + o2.to_i).should == 676
-    end
-
-  end
-
-  describe "multi-process fuzzing", slow: true, java: false do
-    before do
-      producer = Poseidon::Producer.new(["localhost:29092"], "my-producer")
-      payload  = "data" * 10
-      100.times do
-        messages = (0...1000).map do |i|
-          Poseidon::MessageToSend.new("slow-topic", payload, i.to_s)
-        end
-        producer.send_messages(messages)
-      end
-    end
-
-    it 'should consume correctly' do
-      read, write = IO.pipe
-      pid1 = fork do
-        group = new_group(64*1024, "slow-topic")
-        10.times do
-          5.times { group.fetch {|_, m| write.write "1:#{m.size}\n" }}
-          sleep(1)
-        end
-      end
-      pid2 = fork do
-        group = new_group(32*1024, "slow-topic")
-        5.times do
-          10.times { group.fetch {|_, m| write.write "2:#{m.size}\n" }}
-          sleep(1)
-        end
-      end
-      pid3 = fork do
-        group = new_group(8*1024, "slow-topic")
-        5.times do
-          50.times { group.fetch {|_, m| write.write "3:#{m.size}\n" }}
-        end
-      end
-      Process.wait(pid2)
-
-      pid4 = fork do
-        group = new_group(8*1024, "slow-topic")
-        5.times do
-          50.times { group.fetch {|_, m| write.write "4:#{m.size}\n" }}
-        end
-      end
-
-      pid5 = fork do
-        group = new_group(32*1024, "slow-topic")
-        10.times do
-          50.times { group.fetch {|_, m| write.write "5:#{m.size}\n" }}
-          sleep(3)
-        end
-      end
-      Process.wait(pid1)
-      Process.wait(pid3)
-      Process.wait(pid4)
-      Process.wait(pid5)
-      write.close
-      raw = read.read
-      read.close
-
-      stats = raw.lines.inject(Hash.new(0)) do |res, line|
-        pid, count = line.chomp.split(":")
-        res[pid.to_i] += count.to_i
-        res
-      end
-      stats.keys.size.should be_within(1).of(4)
-      stats.values.inject(0, :+).should == 100_000
-    end
-
-  end
-end