racecar 2.4.0 → 2.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9bb07a3f984de71b18b9401ec7908158aba831c54a4972e9de7af59d2d7ae3ea
-  data.tar.gz: 6e00e2f40495624944963868001e0cf20885c9f33e49ad8d147d0037562e96fa
+  metadata.gz: 4cbf6ca4837bdd3893c8342936f9d5ea115a5b3a585e0c843e62171e00038e72
+  data.tar.gz: d8603c46cf801d0f41f017d5b27b35e0d2b195eec5be11a1f7b6bf77413726b4
 SHA512:
-  metadata.gz: 0f5d1b6b84dbd96343a09387709676e0ac1250f1ac79f82225cb2060d8900f91f724f1bf0c722d6c440ce6c65ee175da930fc0bd542f6138eeeaab7d80d9662a
-  data.tar.gz: ca635a04ca6ea5019e625563417ac5c3097d2999d145a26089a9f86eb00312cf242110dcf44958180bdafbfebac5a319fb70b559d2cebe364c906caa93db9ced
+  metadata.gz: 78bdedfe7bd888a3716f5a29f28810f4c4d2f03e43c33b8e47ed08607155211fca1d47f4eab6a4f0def6976bb7ca3263117e52d2e0fa7f0a721c56ec70c7ee3d
+  data.tar.gz: 2bf6e2b50980add68363b59718f54403caada58abcb404a9018ac4304519bbbb7ed920578d37bcadefad63964812e7374d367298f403577725eef79af269c604

data/.github/dependabot.yml

@@ -0,0 +1,17 @@
+version: 2
+registries:
+  artifactory:
+    type: rubygems-server
+    url: ${{secrets.ARTIFACTORY_URL}}
+    username: ${{secrets.ARTIFACTORY_USERNAME}}
+    password: ${{secrets.ARTIFACTORY_API_KEY}}
+updates:
+  - package-ecosystem: "bundler"
+    vendor: true
+    directory: "/"
+    registries:
+      - artifactory
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-type: "direct" # only explicitly defined dependencies.

data/.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        ruby-version: ["2.4", "2.5", "2.6", "3.0"]
+        ruby-version: ["2.6", "3.0"]
 
     steps:
     - uses: zendesk/checkout@v2

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## Unreleased
+
+## v2.8.2
+* Handles ErroneousStateError, in previous versions the consumer would do several unecessary group leave/joins. The log level is also changed to WARN instead of ERROR. ([#285](https://github.com/zendesk/racecar/pull/295))
+
+## v2.8.1
+* Adds new ErroneousStateError to racecar in order to give more information on this new possible exception.
+
+## v2.8.0
+* Update librdkafka version from 1.8.2 to 1.9.0 by upgrading from rdkafka 0.10.0 to 0.12.0. ([#283](https://github.com/zendesk/racecar/pull/293))
+
+## v2.7.0
+
+* Update librdkafka version from 1.5.0 to 1.8.2 by upgrading from rdkafka 0.10.0 to 0.11.1. ([#283](https://github.com/zendesk/racecar/pull/283))
+* Bumps minimum required Ruby version to 2.6
+
+## v2.6.0
+
+* Add capability to specify partition number when producing messages
+
+## v2.5.0
+
+* `fetch_messages` can be configured per consumer, just as the maximum timeout to wait for a full batch.
+
 ## v2.4.0
 
 * Update librdkafka version from 1.4.0 to 1.5.0 by upgrading from rdkafka 0.8.0 to 0.10.0. ([#263](https://github.com/zendesk/racecar/pull/263))

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    racecar (2.3.1)
+    racecar (2.8.2.beta)
       king_konf (~> 1.0.0)
-      rdkafka (~> 0.10.0)
+      rdkafka (~> 0.12.0)
 
 GEM
   remote: https://rubygems.org/
@@ -16,22 +16,22 @@ GEM
     coderay (1.1.3)
     concurrent-ruby (1.1.9)
     diff-lcs (1.4.4)
-    dogstatsd-ruby (4.8.2)
-    ffi (1.15.4)
+    dogstatsd-ruby (5.2.0)
+    ffi (1.15.5)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     king_konf (1.0.0)
     method_source (1.0.0)
-    mini_portile2 (2.7.0)
+    mini_portile2 (2.8.0)
     minitest (5.14.4)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     rake (13.0.1)
-    rdkafka (0.10.0)
-      ffi (~> 1.9)
-      mini_portile2 (~> 2.1)
-      rake (>= 12.3)
+    rdkafka (0.12.0)
+      ffi (~> 1.15)
+      mini_portile2 (~> 2.6)
+      rake (> 12)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
@@ -56,7 +56,7 @@ PLATFORMS
 DEPENDENCIES
   activesupport (< 6.0)
   bundler (>= 1.13, < 3)
-  dogstatsd-ruby (>= 4.0.0, < 5.0.0)
+  dogstatsd-ruby (>= 4.0.0, < 6.0.0)
   pry
   racecar!
   rake (> 10.0)
@@ -64,4 +64,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   2.2.15
+   2.3.7

data/README.md

@@ -269,6 +269,10 @@ end
 - `group_id` – The group id to use for a given group of consumers. Note that this _must_ be different for each consumer class. If left blank a group id is generated based on the consumer class name such that (for example) a consumer with the class name `BaconConsumer` would default to a group id of `bacon-consumer`.
 - `group_id_prefix` – A prefix used when generating consumer group names. For instance, if you set the prefix to be `kevin.` and your consumer class is named `BaconConsumer`, the resulting consumer group will be named `kevin.bacon-consumer`.
 
+#### Batches
+
+- `fetch_messages` - The number of messages to fetch in a single batch. This can be set on a per consumer basis.
+
 #### Logging
 
 - `logfile` – A filename that log messages should be written to. Default is `nil`, which means logs will be written to standard output.

data/lib/racecar/config.rb

@@ -223,6 +223,7 @@ module Racecar
       self.parallel_workers = consumer_class.parallel_workers
       self.subscriptions = consumer_class.subscriptions
       self.max_wait_time = consumer_class.max_wait_time || self.max_wait_time
+      self.fetch_messages = consumer_class.fetch_messages || self.fetch_messages
       self.pidfile ||= "#{group_id}.pid"
     end
 

data/lib/racecar/consumer.rb

@@ -9,7 +9,7 @@ module Racecar
     class << self
       attr_accessor :max_wait_time
       attr_accessor :group_id
-      attr_accessor :producer, :consumer, :parallel_workers
+      attr_accessor :producer, :consumer, :parallel_workers, :fetch_messages
 
       def subscriptions
         @subscriptions ||= []
@@ -93,13 +93,14 @@ module Racecar
     protected
 
     # https://github.com/appsignal/rdkafka-ruby#producing-messages
-    def produce(payload, topic:, key: nil, partition_key: nil, headers: nil, create_time: nil)
+    def produce(payload, topic:, key: nil, partition: nil, partition_key: nil, headers: nil, create_time: nil)
       @delivery_handles ||= []
       message_size = payload.respond_to?(:bytesize) ? payload.bytesize : 0
       instrumentation_payload = {
         value: payload,
         headers: headers,
         key: key,
+        partition: partition,
         partition_key: partition_key,
         topic: topic,
         message_size: message_size,
@@ -112,6 +113,7 @@ module Racecar
           topic: topic,
           payload: payload,
           key: key,
+          partition: partition,
           partition_key: partition_key,
           timestamp: create_time,
           headers: headers,

data/lib/racecar/consumer_set.rb

@@ -49,6 +49,12 @@ module Racecar
 
     def store_offset(message)
       current.store_offset(message)
+    rescue Rdkafka::RdkafkaError => e
+      if e.code == :state # -172
+        @logger.warn "Attempted to store_offset, but we're not subscribed to it: #{ErroneousStateError.new(e)}"
+        return
+      end
+      raise e
     end
 
     def commit

data/lib/racecar/datadog.rb

@@ -63,10 +63,14 @@ module Racecar
         clear
       end
 
+      def close
+        @statsd&.close
+      end
+
       private
 
       def clear
-        @statsd && @statsd.close
+        close
         @statsd = nil
       end
     end

data/lib/racecar/erroneous_state_error.rb

@@ -0,0 +1,34 @@
+# `rd_kafka_offsets_store()` (et.al) returns an error for any
+# partition that is not currently assigned (through `rd_kafka_*assign()`).
+# This prevents a race condition where an application would store offsets
+# after the assigned partitions had been revoked (which resets the stored
+# offset), that could cause these old stored offsets to be committed later
+# when the same partitions were assigned to this consumer again - effectively
+# overwriting any committed offsets by any consumers that were assigned the
+# same partitions previously. This would typically result in the offsets
+# rewinding and messages to be reprocessed.
+# As an extra effort to avoid this situation the stored offset is now
+# also reset when partitions are assigned (through `rd_kafka_*assign()`).
+module Racecar
+  class ErroneousStateError < StandardError
+    def initialize(rdkafka_error)
+      raise rdkafka_error unless rdkafka_error.is_a?(Rdkafka::RdkafkaError)
+
+      @rdkafka_error = rdkafka_error
+    end
+
+    attr_reader :rdkafka_error
+
+    def code
+      @rdkafka_error.code
+    end
+
+    def to_s
+      <<~EOM
+        Partition is no longer assigned to this consumer and the offset could not be stored for commit.
+        #{@rdkafka_error.to_s}
+      EOM
+    end
+
+  end
+end

data/lib/racecar/runner.rb

@@ -4,6 +4,7 @@ require "rdkafka"
 require "racecar/pause"
 require "racecar/message"
 require "racecar/message_delivery_error"
+require "racecar/erroneous_state_error"
 
 module Racecar
   class Runner
@@ -92,6 +93,7 @@ module Racecar
       end
     ensure
       producer.close
+      Racecar::Datadog.close if Object.const_defined?("Racecar::Datadog")
     end
 
     def stop

data/lib/racecar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Racecar
-  VERSION = "2.4.0"
+  VERSION = "2.8.2"
 end

data/racecar.gemspec

@@ -20,16 +20,16 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = '>= 2.4'
+  spec.required_ruby_version = '>= 2.6'
 
   spec.add_runtime_dependency "king_konf", "~> 1.0.0"
-  spec.add_runtime_dependency "rdkafka",   "~> 0.10.0"
+  spec.add_runtime_dependency "rdkafka",   "~> 0.12.0"
 
   spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake", "> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "timecop"
-  spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 5.0.0"
+  spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 6.0.0"
   spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.8.2
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-09-20 00:00:00.000000000 Z
+date: 2022-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 0.12.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 0.12.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -124,7 +124,7 @@ dependencies:
         version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 6.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -134,7 +134,7 @@ dependencies:
         version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 6.0.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -166,6 +166,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
+- ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
@@ -199,6 +200,7 @@ files:
 - lib/racecar/ctl.rb
 - lib/racecar/daemon.rb
 - lib/racecar/datadog.rb
+- lib/racecar/erroneous_state_error.rb
 - lib/racecar/heroku.rb
 - lib/racecar/instrumenter.rb
 - lib/racecar/message.rb
@@ -222,14 +224,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A framework for running Kafka consumers