racecar 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76a6192bd54678ebddbe84a29ab8f1cb8fdc3f40d30f1732bb9426eb28fa14d7
-  data.tar.gz: af3bfc779898a1ccbd7ff862029e3b2c988fa6d07f58648e4b0d22ea8fb7752d
+  metadata.gz: 03f0ef60cb9d16c40adc14765b5fe497b1a1ee5384e7c63d4941373a618c5b52
+  data.tar.gz: 8551c7ec9b838ca3df774afb9f5e886957b3ac7ae3d88f63fc64b084aee03e9a
 SHA512:
-  metadata.gz: 4ea1522b70bb40ef6f0cda810bf0c214bcc3f85a28945f3d47e0bbf443824ea5b5cddffe1e4fd5f948dc999ddf2350425404fe586716d1cc5ce74bbf4adfc29d
-  data.tar.gz: 4af6a0eb25673036ae79f23b01bab4ac641aa2f57dbad33e6ed63bdb47ca1657135770ca7d9a52a7efd12b880e7c2b27bdfa05379c14345a5c9301e6f5c0e9e7
+  metadata.gz: afc4e4df8135bb55c109213ba09bd05c258ceccfb7674530a8ca9a670cee6bbb962bd3994d73682257a2ea530d7a8727b2b20c4a33e2ac5dbe229783768302a6
+  data.tar.gz: 3228f755bab3981835922c4dd47f3d25c65b258ff1fd612b9d752f7360bcb2f0613b9d3d455aeed66f45f39d7ce5ca2fd4732e8e116c4e61b26a9a25f2ab0b76

data/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+## racecar v0.4.1
+
+* Allow configuring the producer (#77).
+* Add support for configuring exponential pause backoff (#76).
+
 ## racecar v0.4.0
 
 * Require Kafka 0.10 or higher.

data/lib/racecar/config.rb

@@ -25,6 +25,12 @@ module Racecar
     desc "How long to pause a partition for if the consumer raises an exception while processing a message -- set to -1 to pause indefinitely"
     float :pause_timeout, default: 10
 
+    desc "When `pause_timeout` and `pause_with_exponential_backoff` are configured, this sets an upper limit on the pause duration"
+    float :max_pause_timeout, default: nil
+
+    desc "Whether to exponentially increase the pause timeout on successive errors -- the timeout is doubled each time"
+    boolean :pause_with_exponential_backoff, default: false
+
     desc "The idle timeout after which a consumer is kicked out of the group"
     float :session_timeout, default: 30
 
@@ -91,6 +97,9 @@ module Racecar
     desc "Run the Racecar process in the background as a daemon"
     boolean :daemonize, default: false
 
+    desc "The codec used to compress messages with"
+    symbol :producer_compression_codec
+
     desc "Enable Datadog metrics"
     boolean :datadog_enabled, default: false
 

data/lib/racecar/runner.rb

@@ -58,7 +58,10 @@ module Racecar
       end
 
       # Configure the consumer with a producer so it can produce messages.
-      producer = kafka.producer
+      producer = kafka.producer(
+        compression_codec: config.producer_compression_codec,
+      )
+
       processor.configure(producer: producer)
 
       begin
@@ -71,10 +74,6 @@ module Racecar
               offset: message.offset,
             }
 
-            # Allow subscribers to receive a notification *before* we process the
-            # message.
-            @instrumenter.instrument("start_process_message.racecar", payload)
-
             @instrumenter.instrument("process_message.racecar", payload) do
               processor.process(message)
               producer.deliver_messages
@@ -90,10 +89,6 @@ module Racecar
               message_count: batch.messages.count,
             }
 
-            # Allow subscribers to receive a notification *before* we process the
-            # message.
-            @instrumenter.instrument("start_process_batch.racecar", payload)
-
             @instrumenter.instrument("process_batch.racecar", payload) do
               processor.process_batch(batch)
               producer.deliver_messages
@@ -110,7 +105,13 @@ module Racecar
           # The partition is automatically resumed after the specified timeout, and will continue where we
           # left off.
           @logger.warn "Pausing partition #{e.topic}/#{e.partition} for #{config.pause_timeout} seconds"
-          consumer.pause(e.topic, e.partition, timeout: config.pause_timeout)
+          consumer.pause(
+            e.topic,
+            e.partition,
+            timeout: config.pause_timeout,
+            max_timeout: config.max_pause_timeout,
+            exponential_backoff: config.pause_with_exponential_backoff?,
+          )
         elsif config.pause_timeout == -1
           # A pause timeout of -1 means indefinite pausing, which in ruby-kafka is done by passing nil as
           # the timeout.

data/lib/racecar/version.rb

@@ -1,3 +1,3 @@
 module Racecar
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

data/racecar.gemspec

@@ -20,8 +20,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "king_konf", "~> 0.2.0"
-  spec.add_runtime_dependency "ruby-kafka", "~> 0.5"
+  spec.add_runtime_dependency "king_konf", "~> 0.3.0"
+  spec.add_runtime_dependency "ruby-kafka", "~> 0.6"
 
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-18 00:00:00.000000000 Z
+date: 2018-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -17,28 +17,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement