sorceror_poseidon_cluster 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9ac4eb0c0f6888a859f4f012f47752acb205d505
+  data.tar.gz: 42f48d6f7a371bd13f4314ae07a264e3e2ed98eb
+SHA512:
+  metadata.gz: 696a4c9c05a130fbefa0cee2d16e4342eceaba6c8b831cbf152ee5b7fe23be50ecd4d518c0b4192976344ba75fd2d60308d849cbe1e59a38beaef795cca198a8
+  data.tar.gz: 102108f9c8a696c3d9e71064459261ce7a08d829211b5518ad41424cffa948589d1ee2bd7348a89df4dbc91e47a15b8180f914dbf87f89cdb8d824ecb65b5e9f

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.gitignore

@@ -0,0 +1,8 @@
+logs/
+kafka*/
+doc/
+.yardoc/
+.bundle/
+pkg/
+coverage/
+*.log

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 2.1.5
+script: bundle exec rake spec:coveralls scenario
+matrix:
+  fast_finish: true

data/Gemfile

@@ -0,0 +1,10 @@
+source "https://rubygems.org"
+
+gemspec
+
+gem 'pry'
+
+platform :jruby do
+  gem 'slyphon-log4j', '= 1.2.15'
+  gem 'slyphon-zookeeper_jar', '= 3.3.5'
+end

data/Gemfile.lock

@@ -0,0 +1,85 @@
+PATH
+  remote: .
+  specs:
+    poseidon_cluster (0.3.0)
+      poseidon (>= 0.0.5.pre1)
+      zk
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.0)
+    coveralls (0.7.0)
+      multi_json (~> 1.3)
+      rest-client
+      simplecov (>= 0.7)
+      term-ansicolor
+      thor
+    diff-lcs (1.2.5)
+    docile (1.1.3)
+    little-plugger (1.1.3)
+    logging (1.8.2)
+      little-plugger (>= 1.1.3)
+      multi_json (>= 1.8.4)
+    method_source (0.8.2)
+    mime-types (2.3)
+    multi_json (1.10.1)
+    poseidon (0.0.5.pre1)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rake (10.3.2)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.0)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-its (1.0.1)
+      rspec-core (>= 2.99.0.beta1)
+      rspec-expectations (>= 2.99.0.beta1)
+    rspec-mocks (3.0.0)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.0)
+    simplecov (0.8.2)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    slop (3.6.0)
+    slyphon-log4j (1.2.15)
+    slyphon-zookeeper_jar (3.3.5-java)
+    term-ansicolor (1.3.0)
+      tins (~> 1.0)
+    thor (0.19.1)
+    tins (1.3.0)
+    yard (0.8.7.4)
+    zk (1.9.4)
+      logging (~> 1.8.2)
+      zookeeper (~> 1.4.0)
+    zookeeper (1.4.9)
+    zookeeper (1.4.9-java)
+      slyphon-log4j (= 1.2.15)
+      slyphon-zookeeper_jar (= 3.3.5)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  bundler
+  coveralls
+  poseidon_cluster!
+  pry
+  rake
+  rspec
+  rspec-its
+  slyphon-log4j (= 1.2.15)
+  slyphon-zookeeper_jar (= 3.3.5)
+  yard

data/README.md

@@ -0,0 +1,95 @@
+# Poseidon Cluster [![Build Status](https://travis-ci.org/bsm/poseidon_cluster.png?branch=master)](https://travis-ci.org/bsm/poseidon_cluster) [![Coverage Status](https://coveralls.io/repos/bsm/poseidon_cluster/badge.png?branch=master)](https://coveralls.io/r/bsm/poseidon_cluster?branch=master)
+
+Poseidon Cluster is a cluster extension of the excellent [Poseidon](http://github.com/bpot/poseidon) Ruby client for Kafka 0.8+. It implements the distribution concept of self-rebalancing *Consumer Groups* and supports the consumption of a single topic from multiple instances.
+
+Consumer group instances share a common group name, and each message published to a topic is delivered to one instance within each subscribing consumer group. Consumer instances can be in separate processes or on separate machines.
+
+## Usage
+
+Launch a consumer group:
+
+```ruby
+require 'poseidon_cluster'
+
+consumer = Poseidon::ConsumerGroup.new(
+            "my-group",                               # Group name
+            ["kafka1.host:9092", "kafka2.host:9092"], # Kafka brokers
+            ["kafka1.host:2181", "kafka2.host:2181"], # Zookeepers hosts
+            "my-topic")                               # Topic name
+
+consumer.partitions # => [0, 1, 2, 3] - all partitions of 'my-topic'
+consumer.claimed    # => [0, 1] - partitions this instance has claimed
+```
+
+Fetch a bulk of messages, auto-commit the offset:
+
+```ruby
+consumer.fetch do |partition, bulk|
+  bulk.each do |m|
+    puts "Fetched '#{m.value}' at #{m.offset} from #{partition}"
+  end
+end
+```
+
+Get the offset for a partition:
+
+```ruby
+consumer.offset(0) # => 320 - current offset from partition 0
+```
+
+Fetch more messages, commit manually:
+
+```ruby
+consumer.fetch commit: false do |partition, bulk|
+  bulk.each do |m|
+    puts "Fetched '#{m.value}' at #{m.offset} from #{partition}"
+  end
+
+  consumer.commit partition, bulk.last.offset+1 unless bulk.empty?
+end
+```
+
+Initiate a fetch-loop, consume indefinitely:
+
+```ruby
+consumer.fetch_loop do |partition, bulk|
+  bulk.each do |m|
+    puts "Fetched '#{m.value}' at #{m.offset} from #{partition}"
+  end
+end
+```
+
+For more details and information, please see the [Poseidon::ConsumerGroup](http://rubydoc.info/github/bsm/poseidon_cluster/Poseidon/ConsumerGroup) documentation and the [Examples](https://github.com/bsm/poseidon_cluster/tree/master/examples).
+
+## Running Tests
+
+The test suite will automatically download, configure and run Kafka locally, you only need a JRE. Run the suite via:
+
+```bash
+bundle exec rake spec
+```
+
+## Licence
+
+```
+Copyright (c) 2014 Black Square Media
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+```

data/Rakefile

@@ -0,0 +1,21 @@
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+require 'yard'
+require 'yard/rake/yardoc_task'
+YARD::Rake::YardocTask.new
+
+require 'coveralls/rake/task'
+Coveralls::RakeTask.new
+namespace :spec do
+  task coveralls: [:spec, 'coveralls:push']
+end
+
+desc "Run full integration test scenario"
+task :scenario do
+  load File.expand_path("../scenario/run.rb", __FILE__)
+end
+
+task default: :spec

data/examples/consumer_group.rb

@@ -0,0 +1,33 @@
+=begin
+
+ PLEASE NOTE
+
+ This example uses threads, but you could equally use fork or run your
+ consumer groups from completely separate process and from multiple machines.
+
+=end
+require 'poseidon_cluster'
+
+# Create a consumer group
+group1 = Poseidon::ConsumerGroup.new "my-group", ["host1:9092", "host2:9092"], ["host1:2181", "host2:2181"], "my-topic"
+
+# Start consuming "my-topic" in a background thread
+thread1 = Thread.new do
+  group1.fetch_loop do |partition, messages|
+    puts "Consumer #1 fetched #{messages.size} from #{partition}"
+  end
+end
+
+# Create a second consumer group
+group2 = Poseidon::ConsumerGroup.new "my-group", ["host1:9092", "host2:9092"], ["host1:2181", "host2:2181"], "my-topic"
+
+# Now consuming all partitions of "my-topic" in parallel
+thread2 = Thread.new do
+  group2.fetch_loop do |partition, messages|
+    puts "Consumer #2 fetched #{messages.size} from #{partition}"
+  end
+end
+
+# Join threads, loop forever
+[thread1, thread2].each(&:join)
+

data/lib/poseidon/cluster.rb

@@ -0,0 +1,28 @@
+require 'socket'
+require 'timeout'
+require 'zk'
+require 'poseidon'
+require 'thread'
+
+module Poseidon::Cluster
+  MAX_INT32 = 0x7fffffff
+  @@sem = Mutex.new
+  @@inc = 0
+
+  # @return [Integer] an incremented number
+  # @api private
+  def self.inc!
+    @@sem.synchronize { @@inc += 1; @@inc = 1 if @@inc > MAX_INT32; @@inc }
+  end
+
+  # @return [String] an globally unique identifier
+  # @api private
+  def self.guid
+    [::Socket.gethostname, ::Process.pid, ::Time.now.to_i, inc!].join("-")
+  end
+
+end
+
+%w|consumer_group|.each do |name|
+  require "poseidon/#{name}"
+end