racecar 0.3.7 → 0.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b82a4c9f1c518dcf39fec9f30661af192aecfcd2
-  data.tar.gz: 7659830d99302a6d4d96e1f13a5822b0485c1b38
+SHA256:
+  metadata.gz: 2e43d9ec7a2fe43e035cb42d1ab21a250072d26bd5dc1b6dde34600870193152
+  data.tar.gz: 4a3dc623057a7c3c78ab5a009a85f9b1bf839d49334578894575a93cb0a06ea9
 SHA512:
-  metadata.gz: 9ccd18859d2f8dd1f96cf9e310fa3677a0338a5f96568fa70d053812bcf3a7f3d2adf2190b1be63a8d86aa57f3898ada337e7bb15979bd456540e8b5a7d14965
-  data.tar.gz: fb5f1e848839852135544d6ad86603288957a40155dd725f68a0d356d61a0fdeb2fa399bd27cb087b2bb89f5f69a24042480d580e07e19d6b44872740083c2d5
+  metadata.gz: 2676b9e5adaa2280b7865b3cc636603ec4a6c827c648b4d9df21796f1891da1f998863c370ebd674ecbf8b0f5b3cf1953e2e521ba09a422c9eed7bfde3d93808
+  data.tar.gz: 59b529314b4b3b13a5ebad8f9a53c51aa24ed4ef8ae22b30f17dfac3474a8dc7714776fcacda29b04b649df4ffee2754c1bd0d1f9da1048b43e3eff864ff1bb1

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+## racecar v0.3.8
+
+* Change the default `max_wait_time` to 1 second.
+* Allow setting the `offset_retention_time` for consumers.
+* Allow pausing partitions indefinitely (#63).
+
 ## racecar v0.3.7
 
 * Allow setting the key and/or partition key when producing messages.

data/README.md

@@ -57,7 +57,7 @@ In Kafka, _consumer groups_ are sets of processes that collaboratively process m
 
 ### Creating consumers
 
-A Racecar consumer is a simple Rails class that inherits from `Racecar::Consumer`:
+A Racecar consumer is a simple Ruby class that inherits from `Racecar::Consumer`:
 
 ```ruby
 class UserBanConsumer < Racecar::Consumer
@@ -231,6 +231,7 @@ The consumers will checkpoint their positions from time to time in order to be a
 
 * `offset_commit_interval` – How often to save the consumer's position in Kafka. Default is every 10 seconds.
 * `offset_commit_threshold` – How many messages to process before forcing a checkpoint. Default is 0, which means there's no limit. Setting this to e.g. 100 makes the consumer stop every 100 messages to checkpoint its position.
+* `offset_retention_time` - How long committed offsets will be retained. Defaults to the broker setting.
 
 #### Timeouts & intervals
 
@@ -241,7 +242,7 @@ All timeouts are defined in number of seconds.
 * `pause_timeout` – How long to pause a partition for if the consumer raises an exception while processing a message. Default is to pause for 10 seconds. Set this to zero in order to disable automatic pausing of partitions.
 * `connect_timeout` – How long to wait when trying to connect to a Kafka broker. Default is 10 seconds.
 * `socket_timeout` – How long to wait when trying to communicate with a Kafka broker. Default is 30 seconds.
-* `max_wait_time` – How long to allow the Kafka brokers to wait before returning messages. A higher number means larger batches, at the cost of higher latency. Default is 5 seconds.
+* `max_wait_time` – How long to allow the Kafka brokers to wait before returning messages. A higher number means larger batches, at the cost of higher latency. Default is 1 second.
 
 #### SSL encryption, authentication & authorization
 
@@ -265,6 +266,16 @@ If using PLAIN:
 * `sasl_plain_username` – The username used to authenticate.
 * `sasl_plain_password` – The password used to authenticate.
 
+#### Datadog monitoring
+
+Racecar supports configuring ruby-kafka's [Datadog](https://www.datadoghq.com/) monitoring integration. If you're running a normal Datadog agent on your host, you just need to set `datadog_enabled` to `true`, as the rest of the settings come with sane defaults.
+
+* `datadog_enabled` – Whether Datadog monitoring is enabled (defaults to `false`).
+* `datadog_host` – The host running the Datadog agent.
+* `datadog_port` – The port of the Datadog agent.
+* `datadog_namespace` – The namespace to use for Datadog metrics.
+* `datadog_tags` – Tags that should always be set on Datadog metrics.
+
 
 ### Testing consumers
 

data/lib/racecar/config.rb

@@ -19,7 +19,10 @@ module Racecar
     desc "How often to send a heartbeat message to Kafka"
     float :heartbeat_interval, default: 10
 
-    desc "How long to pause a partition for if the consumer raises an exception while processing a message"
+    desc "How long committed offsets will be retained."
+    integer :offset_retention_time
+
+    desc "How long to pause a partition for if the consumer raises an exception while processing a message -- set to -1 to pause indefinitely"
     float :pause_timeout, default: 10
 
     desc "The idle timeout after which a consumer is kicked out of the group"
@@ -32,7 +35,7 @@ module Racecar
     float :socket_timeout, default: 30
 
     desc "How long to allow the Kafka brokers to wait before returning messages"
-    float :max_wait_time, default: 5
+    float :max_wait_time, default: 1
 
     desc "A prefix used when generating consumer group names"
     string :group_id_prefix
@@ -139,6 +142,7 @@ module Racecar
 
       self.subscriptions = consumer_class.subscriptions
       self.max_wait_time = consumer_class.max_wait_time || self.max_wait_time
+      self.offset_retention_time = consumer_class.offset_retention_time || self.offset_retention_time
       self.pidfile ||= "#{group_id}.pid"
     end
 

data/lib/racecar/consumer.rb

@@ -5,6 +5,7 @@ module Racecar
     class << self
       attr_accessor :max_wait_time
       attr_accessor :group_id
+      attr_accessor :offset_retention_time
 
       def subscriptions
         @subscriptions ||= []

data/lib/racecar/runner.rb

@@ -35,6 +35,7 @@ module Racecar
         offset_commit_threshold: config.offset_commit_threshold,
         session_timeout: config.session_timeout,
         heartbeat_interval: config.heartbeat_interval,
+        offset_retention_time: config.offset_retention_time,
       )
 
       # Stop the consumer on SIGINT, SIGQUIT or SIGTERM.
@@ -107,6 +108,11 @@ module Racecar
           # left off.
           @logger.warn "Pausing partition #{e.topic}/#{e.partition} for #{config.pause_timeout} seconds"
           consumer.pause(e.topic, e.partition, timeout: config.pause_timeout)
+        elsif config.pause_timeout == -1
+          # A pause timeout of -1 means indefinite pausing, which in ruby-kafka is done by passing nil as
+          # the timeout.
+          @logger.warn "Pausing partition #{e.topic}/#{e.partition} indefinitely, or until the process is restarted"
+          consumer.pause(e.topic, e.partition, timeout: nil)
         end
 
         config.error_handler.call(e.cause, {

data/lib/racecar/version.rb

@@ -1,3 +1,3 @@
 module Racecar
-  VERSION = "0.3.7"
+  VERSION = "0.3.8"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.3.8
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-15 00:00:00.000000000 Z
+date: 2018-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A framework for running Kafka consumers