ruby-kafka 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1724a8fa55a7d963627724e1bd039fa27ebe4ee5
-  data.tar.gz: fe4d190b76588cbd7aaedc806e72e75a23a477d5
+  metadata.gz: f399830029bb1bf2e51d01d01c1ade3015b216c2
+  data.tar.gz: c527a2ddd56a1e4d9e7399727b1dcc4d9a56efa4
 SHA512:
-  metadata.gz: b767bfdada39400fc1bd47987bed986dca5acd7e0ff4bc4456a307b185479bb60205d104829b90cccd5ac1c8ee96209da36490ca9ee5379c2456c3e0ba8b34f8
-  data.tar.gz: 8b120817c769e5dd8b05673695be42581a12f9a5a18c0d5a0040916bd53d2df8c72544b35e84f1fecd262d347b6ddf5f7fbe084ca2e855272e86c775a8d5b7c4
+  metadata.gz: 7f8f5ce7b095ba1dee302e8ffc0a7a23208e66ef6cdcd9f8c39cba753808c7ec741014c593deb41b936219d39c2841998f34c3c3e05bf498bd052721f2eaff78
+  data.tar.gz: 46987298c959729ea40693c07cf3a1e71c2c43dda70141d64870393ef67fc95ec4d8c8fd1443ddf581edb2b022faf6398deaaeb91440ed3cc4871581cf71b391

data/.gitignore

@@ -8,3 +8,4 @@
 /spec/reports/
 /tmp/
 .env
+*.log

data/CHANGELOG.md

@@ -4,6 +4,10 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.3.7
+
+- Default to port 9092 if no port is provided for a seed broker.
+
 ## v0.3.6
 
 - Fix bug that caused partition information to not be reliably updated.

data/README.md

@@ -378,6 +378,22 @@ Each consumer process will be assigned one or more partitions from each topic th
 
 In order to be able to resume processing after a consumer crashes, each consumer will periodically _checkpoint_ its position within each partition it reads from. Since each partition has a monotonically increasing sequence of message offsets, this works by _committing_ the offset of the last message that was processed in a given partition. Kafka handles these commits and allows another consumer in a group to resume from the last commit when a member crashes or becomes unresponsive.
 
+By default, offsets are committed every 10 seconds. You can increase the frequency, known as the _offset commit interval_, to limit the duration of double-processing scenarios, at the cost of a lower throughput due to the added coordination. If you want to improve throughtput, and double-processing is of less concern to you, then you can decrease the frequency.
+
+In addition to the time based trigger it's possible to trigger checkpointing in response to _n_ messages having been processed, known as the _offset commit threshold_. This puts a bound on the number of messages that can be double-processed before the problem is detected. Setting this to 1 will cause an offset commit to take place every time a message has been processed. By default this trigger is disabled.
+
+```ruby
+consumer = kafka.consumer(
+  group_id: "some-group",
+
+  # Increase offset commit frequency to once every 5 seconds.
+  offset_commit_interval: 5,
+
+  # Commit offsets when 100 messages have been processed.
+  offset_commit_threshold: 100,
+)
+```
+
 
 #### Consuming Messages in Batches
 

data/circle.yml

@@ -1,3 +1,19 @@
+machine:
+  services:
+    - docker
+  environment:
+    LOG_TO_STDERR: true
+    LOG_LEVEL: DEBUG
+
 dependencies:
   pre:
+    - docker -v
+    - docker pull ches/kafka:0.9.0.1
+    - docker pull jplock/zookeeper:3.4.6
     - gem install bundler -v 1.9.5
+
+test:
+  override:
+    - bundle exec rspec
+  post:
+    - cp *.log $CIRCLE_ARTIFACTS/ || true

data/lib/kafka/cluster.rb

@@ -137,6 +137,7 @@ module Kafka
 
         begin
           host, port = node.split(":", 2)
+          port ||= 9092 # Default Kafka port.
 
           broker = @broker_pool.connect(host, port.to_i)
           cluster_info = broker.fetch_metadata(topics: @target_topics)

data/lib/kafka/connection.rb

@@ -77,6 +77,7 @@ module Kafka
     def send_request(request)
       # Default notification payload.
       notification = {
+        broker_host: @host,
         api: Protocol.api_name(request.api_key),
         request_size: 0,
         response_size: 0,

data/lib/kafka/consumer.rb

@@ -115,10 +115,10 @@ module Kafka
               yield message
             end
 
+            mark_message_as_processed(message)
             @offset_manager.commit_offsets_if_necessary
 
             @heartbeat.send_if_necessary
-            mark_message_as_processed(message)
 
             return if !@running
           end

data/lib/kafka/offset_manager.rb

@@ -42,6 +42,7 @@ module Kafka
         @last_commit = Time.now
 
         @uncommitted_offsets = 0
+        @committed_offsets = nil
       end
     end
 

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.6"
+  VERSION = "0.3.7"
 end

data/ruby-kafka.gemspec

@@ -36,4 +36,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec-benchmark"
   spec.add_development_dependency "activesupport", ">= 4.2.0", "< 5.1"
   spec.add_development_dependency "snappy"
+  spec.add_development_dependency "colored"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-02 00:00:00.000000000 Z
+date: 2016-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -142,6 +142,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: colored
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |-
   A client library for the Kafka distributed commit log.