racecar 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 03f0ef60cb9d16c40adc14765b5fe497b1a1ee5384e7c63d4941373a618c5b52
-  data.tar.gz: 8551c7ec9b838ca3df774afb9f5e886957b3ac7ae3d88f63fc64b084aee03e9a
+  metadata.gz: c45d8677d5b6355960cdef16cddc745909b28145bb3243cfa64170f6f284609a
+  data.tar.gz: dfed28e1acb40d781aedc6c0ae1d6d916b04efa48919cad6a05c4310720fee0b
 SHA512:
-  metadata.gz: afc4e4df8135bb55c109213ba09bd05c258ceccfb7674530a8ca9a670cee6bbb962bd3994d73682257a2ea530d7a8727b2b20c4a33e2ac5dbe229783768302a6
-  data.tar.gz: 3228f755bab3981835922c4dd47f3d25c65b258ff1fd612b9d752f7360bcb2f0613b9d3d455aeed66f45f39d7ce5ca2fd4732e8e116c4e61b26a9a25f2ab0b76
+  metadata.gz: 5a8f1c5077c03fb3dcca7124d6465ba6386f5a9a0dbe7ef5bf752073c58f04cc36c2665c1b5595b692289dfacddbb83e5cc022f07fbf05fc83dfcca266e743fb
+  data.tar.gz: 75dc3b0d158f0a4b8fb8a8242dc7f4b2c043217792f6351a27d2a8698361e7e86fe04315a4b167202eb1703045b58183533019ef9dd48563db954c076ddb4b01

data/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+## racecar v0.4.2
+
+* Allow configuring `max_bytes` and `max_fetch_queue_size`.
+
 ## racecar v0.4.1
 
 * Allow configuring the producer (#77).

data/README.md

@@ -246,6 +246,17 @@ All timeouts are defined in number of seconds.
 * `socket_timeout` – How long to wait when trying to communicate with a Kafka broker. Default is 30 seconds.
 * `max_wait_time` – How long to allow the Kafka brokers to wait before returning messages. A higher number means larger batches, at the cost of higher latency. Default is 1 second.
 
+#### Memory & network usage
+
+Kafka is _really_ good at throwing data at consumers, so you may want to tune these variables in order to avoid ballooning your process' memory or saturating your network capacity.
+
+Racecar uses ruby-kafka under the hood, which fetches messages from the Kafka brokers in a background thread. This thread pushes fetch responses, possible containing messages from many partitions, into a queue that is read by the processing thread (AKA your code). The main way to control the fetcher thread is to control the size of those responses and the size of the queue.
+
+* `max_bytes` — The maximum size of message sets returned from a single fetch request.
+* `max_fetch_queue_size` — The maximum number of fetch responses to keep in the queue.
+
+The memory usage limit is roughly estimated as `max_bytes * max_fetch_queue_size`, plus whatever your application uses.
+
 #### SSL encryption, authentication & authorization
 
 * `ssl_ca_cert` – A valid SSL certificate authority, as a string.
@@ -274,6 +285,12 @@ If using SCRAM:
 * `sasl_scram_password` – The password used to authenticate.
 * `sasl_scram_mechanism` – The SCRAM mechanism to use, either `sha256` or `sha512`.
 
+#### Producing messages
+
+These settings are related to consumers that _produce messages to Kafka_.
+
+* `producer_compression_codec` – If defined, Racecar will compress messages before writing them to Kafka. The codec needs to be one of `gzip`, `lz4`, or `snappy`, either as a Symbol or a String.
+
 #### Datadog monitoring
 
 Racecar supports configuring ruby-kafka's [Datadog](https://www.datadoghq.com/) monitoring integration. If you're running a normal Datadog agent on your host, you just need to set `datadog_enabled` to `true`, as the rest of the settings come with sane defaults.

data/lib/racecar/cli.rb

@@ -64,6 +64,8 @@ module Racecar
     rescue => e
       $stderr.puts "=> Crashed: #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
 
+      config.error_handler.call(e)
+
       raise
     end
 

data/lib/racecar/config.rb

@@ -22,6 +22,9 @@ module Racecar
     desc "How long committed offsets will be retained."
     integer :offset_retention_time
 
+    desc "The maximum number of fetch responses to keep queued before processing"
+    integer :max_fetch_queue_size, default: 10
+
     desc "How long to pause a partition for if the consumer raises an exception while processing a message -- set to -1 to pause indefinitely"
     float :pause_timeout, default: 10
 
@@ -43,6 +46,9 @@ module Racecar
     desc "How long to allow the Kafka brokers to wait before returning messages"
     float :max_wait_time, default: 1
 
+    desc "The maximum size of message sets returned from a single fetch"
+    integer :max_bytes, default: 10485760
+
     desc "A prefix used when generating consumer group names"
     string :group_id_prefix
 
@@ -67,6 +73,9 @@ module Racecar
     desc "A valid SSL client certificate key"
     string :ssl_client_cert_key
 
+    desc "Support for using the CA certs installed on your system by default for SSL. More info, see: https://github.com/zendesk/ruby-kafka/pull/521"
+    boolean :ssl_ca_certs_from_system, default: false
+
     desc "The GSSAPI principal"
     string :sasl_gssapi_principal
 
@@ -146,6 +155,10 @@ module Racecar
       if connect_timeout <= max_wait_time
         raise ConfigError, "`connect_timeout` must be longer than `max_wait_time`"
       end
+
+      if max_pause_timeout && !pause_with_exponential_backoff?
+        raise ConfigError, "`max_pause_timeout` only makes sense when `pause_with_exponential_backoff` is enabled"
+      end
     end
 
     def load_consumer_class(consumer_class)

data/lib/racecar/consumer.rb

@@ -13,10 +13,14 @@ module Racecar
 
       # Adds one or more topic subscriptions.
       #
-      # start_from_beginning    - whether to start from the beginning or the end of each
-      #                           partition.
-      # max_bytes_per_partition - the maximum number of bytes to fetch from each partition
-      #                           at a time.
+      # Can be called multiple times in order to subscribe to more topics.
+      #
+      # @param topics [String] one or more topics to subscribe to.
+      # @param start_from_beginning [Boolean] whether to start from the beginning or the end
+      #   of each partition.
+      # @param max_bytes_per_partition [Integer] the maximum number of bytes to fetch from
+      #   each partition at a time.
+      # @return [nil]
       def subscribes_to(*topics, start_from_beginning: true, max_bytes_per_partition: 1048576)
         topics.each do |topic|
           subscriptions << Subscription.new(topic, start_from_beginning, max_bytes_per_partition)

data/lib/racecar/runner.rb

@@ -30,6 +30,7 @@ module Racecar
         sasl_scram_username: config.sasl_scram_username,
         sasl_scram_password: config.sasl_scram_password,
         sasl_scram_mechanism: config.sasl_scram_mechanism,
+        ssl_ca_certs_from_system: config.ssl_ca_certs_from_system,
       )
 
       @consumer = kafka.consumer(
@@ -39,6 +40,7 @@ module Racecar
         session_timeout: config.session_timeout,
         heartbeat_interval: config.heartbeat_interval,
         offset_retention_time: config.offset_retention_time,
+        fetcher_max_queue_size: config.max_fetch_queue_size,
       )
 
       # Stop the consumer on SIGINT, SIGQUIT or SIGTERM.
@@ -66,7 +68,7 @@ module Racecar
 
       begin
         if processor.respond_to?(:process)
-          consumer.each_message(max_wait_time: config.max_wait_time) do |message|
+          consumer.each_message(max_wait_time: config.max_wait_time, max_bytes: config.max_bytes) do |message|
             payload = {
               consumer_class: processor.class.to_s,
               topic: message.topic,
@@ -80,7 +82,7 @@ module Racecar
             end
           end
         elsif processor.respond_to?(:process_batch)
-          consumer.each_batch(max_wait_time: config.max_wait_time) do |batch|
+          consumer.each_batch(max_wait_time: config.max_wait_time, max_bytes: config.max_bytes) do |batch|
             payload = {
               consumer_class: processor.class.to_s,
               topic: batch.topic,

data/lib/racecar/version.rb

@@ -1,3 +1,3 @@
 module Racecar
-  VERSION = "0.4.1"
+  VERSION = "0.4.2"
 end

data/racecar.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "king_konf", "~> 0.3.0"
+  spec.add_runtime_dependency "king_konf", "~> 0.3.6"
   spec.add_runtime_dependency "ruby-kafka", "~> 0.6"
 
   spec.add_development_dependency "bundler", "~> 1.13"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-05-17 00:00:00.000000000 Z
+date: 2018-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 0.3.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 0.3.6
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement