racecar 2.3.1 → 2.7.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76c24f3e555a6dc918ac8305af887ffef0ac01a69777a23b87f82ceffde2ac10
-  data.tar.gz: d726316346947532195ee496a6968d7805a488d79e365e729bc5402b8fc72020
+  metadata.gz: 5fa253ad86ec573e727e7eb156b0a48f464167553966ac02f7765bc50db6c80e
+  data.tar.gz: c780b364b6ef7f54740fddeb38b13d835c922671b04c2a71e24135ea23977606
 SHA512:
-  metadata.gz: 26717684ff1357eea174e0da260242aaa2dbc5f9114e3b9a740f76c905e42ee2a9e888dbe8baf9fb03df4a493ce9f310d0a38e296cf0810c94c378627d0f96bc
-  data.tar.gz: 1cfabc0277274391c0ffe58d4ad6e222de078e526e581eb262e32b1b4c23dbb76713f2a39649d8e4578c5d141bf9c42c9700c2de638dbc3dc23895f99a53c66b
+  metadata.gz: 99a36891132422bbb90165e5f9d0eca25076d8c85b128ee4d28057d678d670bd4ec6e8a2c83cf1517a62a5dc2536aabc4f01ed8b51f67a08941087ab64cfb472
+  data.tar.gz: 3be8df1ebfec2c317346e202f8d868b2d30d361885a71a17484ab9a2b3e52e1f3ca8f9cd61215f3e89809aa258dcca67ec3b0b714a74416f882e33096b9f0f8c

data/.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        ruby-version: ["2.5", "2.6", "3.0"]
+        ruby-version: ["2.6", "3.0"]
 
     steps:
     - uses: zendesk/checkout@v2

data/CHANGELOG.md

@@ -2,6 +2,23 @@
 
 ## Unreleased
 
+## v2.7.0
+
+* Update librdkafka version from 1.5.0 to 1.8.2 by upgrading from rdkafka 0.10.0 to 0.11.1. ([#283](https://github.com/zendesk/racecar/pull/283))
+
+## v2.6.0
+
+* Add capability to specify partition number when producing messages
+
+## v2.5.0
+
+* `fetch_messages` can be configured per consumer, just as the maximum timeout to wait for a full batch.
+
+## v2.4.0
+
+* Update librdkafka version from 1.4.0 to 1.5.0 by upgrading from rdkafka 0.8.0 to 0.10.0. ([#263](https://github.com/zendesk/racecar/pull/263))
+* Restore support for Ruby 2.4 (#258)
+
 ## racecar v2.3.1
 
 * Handle `ERR_NOT_COORDINATOR` (#209)

data/Gemfile

@@ -4,3 +4,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in racecar.gemspec
 gemspec
+
+# We actually support version 6.0 (see gemspec); this extra restriction is added just for running the test suite also
+# on Ruby 2.4, which activesupport 6.0 no longer supports
+gem 'activesupport', '< 6.0'

data/Gemfile.lock

@@ -1,38 +1,37 @@
 PATH
   remote: .
   specs:
-    racecar (2.3.0)
+    racecar (2.6.0)
       king_konf (~> 1.0.0)
-      rdkafka (~> 0.8.0)
+      rdkafka (~> 0.11.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.4)
+    activesupport (5.2.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
     coderay (1.1.3)
-    concurrent-ruby (1.1.7)
+    concurrent-ruby (1.1.9)
     diff-lcs (1.4.4)
-    dogstatsd-ruby (4.8.2)
-    ffi (1.15.3)
-    i18n (1.8.5)
+    dogstatsd-ruby (5.2.0)
+    ffi (1.15.5)
+    i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     king_konf (1.0.0)
     method_source (1.0.0)
-    mini_portile2 (2.6.1)
-    minitest (5.14.2)
+    mini_portile2 (2.7.1)
+    minitest (5.14.4)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     rake (13.0.1)
-    rdkafka (0.8.1)
-      ffi (~> 1.9)
-      mini_portile2 (~> 2.1)
-      rake (>= 12.3)
+    rdkafka (0.11.1)
+      ffi (~> 1.15)
+      mini_portile2 (~> 2.6)
+      rake (> 12)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
@@ -48,17 +47,16 @@ GEM
     rspec-support (3.10.2)
     thread_safe (0.3.6)
     timecop (0.9.2)
-    tzinfo (1.2.8)
+    tzinfo (1.2.9)
       thread_safe (~> 0.1)
-    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport (>= 4.0, < 6.1)
+  activesupport (< 6.0)
   bundler (>= 1.13, < 3)
-  dogstatsd-ruby (>= 4.0.0, < 5.0.0)
+  dogstatsd-ruby (>= 4.0.0, < 6.0.0)
   pry
   racecar!
   rake (> 10.0)
@@ -66,4 +64,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   2.1.4
+   2.3.7

data/README.md

@@ -269,6 +269,10 @@ end
 - `group_id` – The group id to use for a given group of consumers. Note that this _must_ be different for each consumer class. If left blank a group id is generated based on the consumer class name such that (for example) a consumer with the class name `BaconConsumer` would default to a group id of `bacon-consumer`.
 - `group_id_prefix` – A prefix used when generating consumer group names. For instance, if you set the prefix to be `kevin.` and your consumer class is named `BaconConsumer`, the resulting consumer group will be named `kevin.bacon-consumer`.
 
+#### Batches
+
+- `fetch_messages` - The number of messages to fetch in a single batch. This can be set on a per consumer basis.
+
 #### Logging
 
 - `logfile` – A filename that log messages should be written to. Default is `nil`, which means logs will be written to standard output.

data/lib/racecar/config.rb

@@ -223,6 +223,7 @@ module Racecar
       self.parallel_workers = consumer_class.parallel_workers
       self.subscriptions = consumer_class.subscriptions
       self.max_wait_time = consumer_class.max_wait_time || self.max_wait_time
+      self.fetch_messages = consumer_class.fetch_messages || self.fetch_messages
       self.pidfile ||= "#{group_id}.pid"
     end
 

data/lib/racecar/consumer.rb

@@ -9,7 +9,7 @@ module Racecar
     class << self
       attr_accessor :max_wait_time
       attr_accessor :group_id
-      attr_accessor :producer, :consumer, :parallel_workers
+      attr_accessor :producer, :consumer, :parallel_workers, :fetch_messages
 
       def subscriptions
         @subscriptions ||= []
@@ -63,25 +63,27 @@ module Racecar
 
         @instrumenter.instrument('deliver_messages', instrumentation_payload) do
           @delivery_handles.each do |handle|
-            # rdkafka-ruby checks every wait_timeout seconds if the message was
-            # successfully delivered, up to max_wait_timeout seconds before raising
-            # Rdkafka::AbstractHandle::WaitTimeoutError. librdkafka will (re)try to
-            # deliver all messages in the background, until "config.message_timeout"
-            # (message.timeout.ms) is exceeded. Phrased differently, rdkafka-ruby's
-            # WaitTimeoutError is just informative.
-            # The raising can be avoided if max_wait_timeout below is greater than
-            # config.message_timeout, but config is not available here (without
-            # changing the interface).
-            handle.wait(max_wait_timeout: 60, wait_timeout: 0.1)
-          rescue Rdkafka::AbstractHandle::WaitTimeoutError => e
-            partition = MessageDeliveryError.partition_from_delivery_handle(handle)
-            # ideally we could use the logger passed to the Runner, but it is not
-            # available here. The runner sets it for Rdkafka, though, so we can use
-            # that instead.
-            @config.logger.debug "Still trying to deliver message to (partition #{partition})... (will try up to Racecar.config.message_timeout)"
-            retry
-          rescue Rdkafka::RdkafkaError => e
-            raise MessageDeliveryError.new(e, handle)
+            begin
+              # rdkafka-ruby checks every wait_timeout seconds if the message was
+              # successfully delivered, up to max_wait_timeout seconds before raising
+              # Rdkafka::AbstractHandle::WaitTimeoutError. librdkafka will (re)try to
+              # deliver all messages in the background, until "config.message_timeout"
+              # (message.timeout.ms) is exceeded. Phrased differently, rdkafka-ruby's
+              # WaitTimeoutError is just informative.
+              # The raising can be avoided if max_wait_timeout below is greater than
+              # config.message_timeout, but config is not available here (without
+              # changing the interface).
+              handle.wait(max_wait_timeout: 60, wait_timeout: 0.1)
+            rescue Rdkafka::AbstractHandle::WaitTimeoutError => e
+              partition = MessageDeliveryError.partition_from_delivery_handle(handle)
+              # ideally we could use the logger passed to the Runner, but it is not
+              # available here. The runner sets it for Rdkafka, though, so we can use
+              # that instead.
+              @config.logger.debug "Still trying to deliver message to (partition #{partition})... (will try up to Racecar.config.message_timeout)"
+              retry
+            rescue Rdkafka::RdkafkaError => e
+              raise MessageDeliveryError.new(e, handle)
+            end
           end
         end
       end
@@ -91,13 +93,14 @@ module Racecar
     protected
 
     # https://github.com/appsignal/rdkafka-ruby#producing-messages
-    def produce(payload, topic:, key: nil, partition_key: nil, headers: nil, create_time: nil)
+    def produce(payload, topic:, key: nil, partition: nil, partition_key: nil, headers: nil, create_time: nil)
       @delivery_handles ||= []
       message_size = payload.respond_to?(:bytesize) ? payload.bytesize : 0
       instrumentation_payload = {
         value: payload,
         headers: headers,
         key: key,
+        partition: partition,
         partition_key: partition_key,
         topic: topic,
         message_size: message_size,
@@ -110,6 +113,7 @@ module Racecar
           topic: topic,
           payload: payload,
           key: key,
+          partition: partition,
           partition_key: partition_key,
           timestamp: create_time,
           headers: headers,

data/lib/racecar/datadog.rb

@@ -63,10 +63,14 @@ module Racecar
         clear
       end
 
+      def close
+        @statsd&.close
+      end
+
       private
 
       def clear
-        @statsd && @statsd.close
+        close
         @statsd = nil
       end
     end

data/lib/racecar/runner.rb

@@ -92,6 +92,7 @@ module Racecar
       end
     ensure
       producer.close
+      Racecar::Datadog.close if Object.const_defined?("Racecar::Datadog")
     end
 
     def stop

data/lib/racecar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Racecar
-  VERSION = "2.3.1"
+  VERSION = "2.7.0.beta1"
 end

data/racecar.gemspec

@@ -20,14 +20,16 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = '>= 2.6'
+
   spec.add_runtime_dependency "king_konf", "~> 1.0.0"
-  spec.add_runtime_dependency "rdkafka",   "~> 0.8.0"
+  spec.add_runtime_dependency "rdkafka",   "~> 0.11.1"
 
   spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake", "> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "timecop"
-  spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 5.0.0"
+  spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 6.0.0"
   spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 2.7.0.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-09-02 00:00:00.000000000 Z
+date: 2022-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.11.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.11.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -124,7 +124,7 @@ dependencies:
         version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 6.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -134,7 +134,7 @@ dependencies:
         version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 6.0.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -222,12 +222,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.1.2
 signing_key: