kafka-consumer 0.0.1

data/test/test_helper.rb

@@ -0,0 +1,48 @@
+require 'minitest/autorun'
+require 'kafka/consumer'
+require 'kazoo'
+require 'mocha/mini_test'
+require 'pp'
+
+module MockCluster
+  def mock_cluster
+    cluster = Kazoo::Cluster.new('example.com:2181')
+    cluster.stubs(:zk).returns(mock)
+
+    cluster.stubs(:brokers).returns(
+      1 => Kazoo::Broker.new(cluster, 1, 'example.com', 9091),
+      2 => Kazoo::Broker.new(cluster, 2, 'example.com', 9092),
+      3 => Kazoo::Broker.new(cluster, 3, 'example.com', 9093),
+    )
+
+    cluster.stubs(:topics).returns(
+      'test.1' => topic_1 = Kazoo::Topic.new(cluster, 'test.1'),
+      'test.4' => topic_4 = Kazoo::Topic.new(cluster, 'test.4')
+    )
+
+    topic_1.stubs(:partitions).returns([
+      Kazoo::Partition.new(topic_1, 0, replicas: [cluster.brokers[1], cluster.brokers[2]]),
+    ])
+
+    topic_4.stubs(:partitions).returns([
+      Kazoo::Partition.new(topic_4, 0, replicas: [cluster.brokers[2], cluster.brokers[1]]),
+      Kazoo::Partition.new(topic_4, 1, replicas: [cluster.brokers[2], cluster.brokers[3]]),
+      Kazoo::Partition.new(topic_4, 2, replicas: [cluster.brokers[1], cluster.brokers[3]]),
+      Kazoo::Partition.new(topic_4, 3, replicas: [cluster.brokers[3], cluster.brokers[2]]),
+    ])
+
+    topic_1.partitions[0].stubs(:isr).returns([cluster.brokers[1], cluster.brokers[2]])
+    topic_4.partitions[0].stubs(:isr).returns([cluster.brokers[2], cluster.brokers[1]])
+    topic_4.partitions[1].stubs(:isr).returns([cluster.brokers[2], cluster.brokers[3]])
+    topic_4.partitions[2].stubs(:isr).returns([cluster.brokers[1], cluster.brokers[3]])
+    topic_4.partitions[3].stubs(:isr).returns([cluster.brokers[3], cluster.brokers[2]])
+
+    topic_1.partitions[0].stubs(:leader).returns(cluster.brokers[1])
+    topic_4.partitions[0].stubs(:leader).returns(cluster.brokers[2])
+    topic_4.partitions[1].stubs(:leader).returns(cluster.brokers[2])
+    topic_4.partitions[2].stubs(:leader).returns(cluster.brokers[1])
+    topic_4.partitions[3].stubs(:leader).returns(cluster.brokers[3])
+
+    return cluster
+  end
+end

data/test/topic_test.rb

@@ -0,0 +1,40 @@
+require 'test_helper'
+
+class TopicTest < Minitest::Test
+  include MockCluster
+
+  def setup
+    @cluster = mock_cluster
+  end
+
+  def test_from_json
+    json_payload = '{"version":1,"partitions":{"2":[1,2,3],"1":[3,1,2],"3":[2,3,1],"0":[3,2,1]}}'
+    topic = Kazoo::Topic.from_json(@cluster, 'test.4', JSON.parse(json_payload))
+
+    assert_equal 4, topic.partitions.length
+    assert_equal [3,2,1], topic.partitions[0].replicas.map(&:id)
+    assert_equal [3,1,2], topic.partitions[1].replicas.map(&:id)
+    assert_equal [1,2,3], topic.partitions[2].replicas.map(&:id)
+    assert_equal [2,3,1], topic.partitions[3].replicas.map(&:id)
+  end
+
+  def test_replication_factor
+    json_payload = '{"version":1,"partitions":{"2":[1,2,3],"1":[3,1,2],"3":[2,3,1],"0":[3,2,1]}}'
+    topic = Kazoo::Topic.from_json(@cluster, 'test.4', JSON.parse(json_payload))
+    assert_equal 3, topic.replication_factor
+
+    json_payload = '{"version":1,"partitions":{"2":[2,3],"1":[2],"3":[2,3,1],"0":[3,2,1]}}'
+    topic = Kazoo::Topic.from_json(@cluster, 'test.4', JSON.parse(json_payload))
+    assert_equal 1, topic.replication_factor
+  end
+
+  def tets_topic_under_replicated?
+    refute @cluster.topics['test.1'].under_replicated?
+    refute @cluster.topics['test.1'].partitions[0].under_replicated?
+
+    @cluster.topics['test.1'].partitions[0].expects(:isr).returns([@cluster.brokers[1]])
+
+    assert @cluster.topics['test.1'].partitions[0].under_replicated?
+    assert @cluster.topics['test.1'].under_replicated?
+  end
+end

metadata

@@ -0,0 +1,161 @@
+--- !ruby/object:Gem::Specification
+name: kafka-consumer
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Willem van Bergen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: poseidon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+- !ruby/object:Gem::Dependency
+  name: zookeeper
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.4'
+description: High-level consumer for Kafka. Implements the Zookeeper-backed consumer
+  implementation that offers offset management, load balancing and automatic failovers.
+email:
+- willem@vanbergen.org
+executables:
+- kazoo
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/kazoo
+- kafka-consumer.gemspec
+- lib/kafka/consumer.rb
+- lib/kafka/consumer/message.rb
+- lib/kafka/consumer/partition_consumer.rb
+- lib/kafka/consumer/version.rb
+- lib/kazoo.rb
+- lib/kazoo/broker.rb
+- lib/kazoo/cli.rb
+- lib/kazoo/cluster.rb
+- lib/kazoo/consumergroup.rb
+- lib/kazoo/partition.rb
+- lib/kazoo/topic.rb
+- lib/kazoo/version.rb
+- test/broker_test.rb
+- test/cluster_test.rb
+- test/partition_distribution_test.rb
+- test/partition_test.rb
+- test/test_helper.rb
+- test/topic_test.rb
+homepage: https://github.com/wvanbergen/kafka-consumer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: High-level consumer for Kafka
+test_files:
+- test/broker_test.rb
+- test/cluster_test.rb
+- test/partition_distribution_test.rb
+- test/partition_test.rb
+- test/test_helper.rb
+- test/topic_test.rb