racecar 2.9.0.beta1 → 2.10.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 364239e804c99b816c0a37802b605ff4035d2d4c51abc0b161e11609268349ea
-  data.tar.gz: 4d7bd65a04b8c914640a2edc11f3c7521b2e061db8eb3c215ba29a785f031e12
+  metadata.gz: e0057ec2861db783b83e34a63c0a09a0be476ed6cb57b760dea3e2cc5ef57d1f
+  data.tar.gz: 22dac9ed21af6f9abfd50f22d9dc86032a68af556e1be77bff2f59cbf505ccf2
 SHA512:
-  metadata.gz: c60037cd0dd5477a77a5fddd42003a3d4be67c38fb1c812cfff698cdb8fb7cd62e7e65097b6508c2a19a768daad5139f3eb469e91f0ef3ff3fe9535f9fd86a8a
-  data.tar.gz: 88d3ffcfa27b6bcb89b454151db353165ab62fa0dd42586c177d05ea6091663227b8d78f14180e69c4937417d0f761fc5666657be1320d44d54eeebcc5d78601
+  metadata.gz: f3baeec4de02a68b55e4a302d2b713de67d0245564aae57cbca363aec8b34633f8dd2220d44c1e212347857d7fde35217644dae8afdbde06103e11a52394997d
+  data.tar.gz: 66a2168ca6f7a4ec7a6d63e8cf767397502c02487d87445b59e4d7fbdd0269abee3279019e36ab1f343e246d4c58146e951ad521b1bc881a4cb60b3f35076b6e

data/.github/workflows/ci.yml

@@ -12,30 +12,34 @@ jobs:
 
     strategy:
       matrix:
-        ruby-version: ["2.6", "3.0"]
+        ruby-version:
+          - "2.7"
+          - "3.0"
+          - "3.1"
+          - "3.2"
 
     steps:
-    - uses: zendesk/checkout@v2
+    - uses: zendesk/checkout@v3
     - name: Set up Ruby
-      uses: zendesk/setup-ruby@v1.64.1
+      uses: zendesk/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true
     - name: Build and test with RSpec
-      run: bundle exec rspec --format documentation --require spec_helper --color --exclude-pattern='spec/integration/*_spec.rb'
+      run: bundle exec rspec --exclude-pattern='spec/integration/*_spec.rb'
 
   integration-specs:
     runs-on: ubuntu-latest
     steps:
-    - uses: zendesk/checkout@v2
+    - uses: zendesk/checkout@v3
     - name: Set up Ruby
-      uses: zendesk/setup-ruby@v1.64.1
+      uses: zendesk/setup-ruby@v1
       with:
-        ruby-version: 2.7
+        ruby-version: "2.7"
         bundler-cache: true
     - name: Bring up docker-compose stack
       run: docker-compose up -d
     - name: Build and test with RSpec
       env:
         RACECAR_BROKERS: localhost:9092
-      run: timeout --kill-after 180 150 bundle exec rspec --format documentation --require spec_helper --color spec/integration/*_spec.rb
+      run: timeout --kill-after 180 150 bundle exec rspec spec/integration/*_spec.rb

data/CHANGELOG.md

@@ -1,6 +1,18 @@
 # Changelog
 
-## v2.9.0.beta1
+## Unreleased
+
+## 2.10.0.beta1
+
+* Bump rdkafka gem version to 0.13.0
+* Support cooperative-sticky
+* Instrument produce delivery errors
+* Fix config load for liveness probe
+* Send exceptions to `process_batch` instrumenter
+* Docker test fixes
+* Test with Ruby 3.2
+
+## v2.9.0, v2.9.0.beta1
 
 * Add `partitioner` producer config option to allow changing the strategy to
   determine which topic partition a message is written to when racecar

data/Dockerfile

@@ -1,4 +1,4 @@
-FROM circleci/ruby:2.7.2
+FROM cimg/ruby:2.7.8
 
 RUN sudo apt-get update
 RUN sudo apt-get install docker

data/Gemfile

@@ -5,6 +5,6 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in racecar.gemspec
 gemspec
 
-# We actually support version 6.0 (see gemspec); this extra restriction is added just for running the test suite also
-# on Ruby 2.4, which activesupport 6.0 no longer supports
-gem 'activesupport', '< 6.0'
+# We actually support version 7.x (see gemspec); this extra restriction is added just for running the test suite also
+# on Ruby 2.6, which activesupport 7.0 does not support.
+gem 'activesupport', '~> 6.1.0'

data/Gemfile.lock

@@ -1,67 +1,72 @@
 PATH
   remote: .
   specs:
-    racecar (2.9.0.beta1)
+    racecar (2.9.0)
       king_konf (~> 1.0.0)
-      rdkafka (~> 0.12.0)
+      rdkafka (~> 0.13.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.6)
+    activesupport (6.1.7.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    byebug (11.1.3)
     coderay (1.1.3)
-    concurrent-ruby (1.1.9)
-    diff-lcs (1.4.4)
-    dogstatsd-ruby (5.2.0)
+    concurrent-ruby (1.2.2)
+    diff-lcs (1.5.0)
+    dogstatsd-ruby (5.5.0)
     ffi (1.15.5)
-    i18n (1.8.10)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     king_konf (1.0.1)
     method_source (1.0.0)
     mini_portile2 (2.8.1)
-    minitest (5.14.4)
-    pry (0.13.1)
+    minitest (5.18.0)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rake (13.0.1)
-    rdkafka (0.12.0)
+    pry-byebug (3.10.1)
+      byebug (~> 11.0)
+      pry (>= 0.13, < 0.15)
+    rake (13.0.6)
+    rdkafka (0.13.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.1)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.4)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    thread_safe (0.3.6)
-    timecop (0.9.2)
-    tzinfo (1.2.10)
-      thread_safe (~> 0.1)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    timecop (0.9.6)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.6.7)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport (< 6.0)
+  activesupport (~> 6.1.0)
   bundler (>= 1.13, < 3)
   dogstatsd-ruby (>= 4.0.0, < 6.0.0)
-  pry
+  pry-byebug
   racecar!
   rake (> 10.0)
   rspec (~> 3.0)
   timecop
 
 BUNDLED WITH
-   2.3.7
+   2.4.9

data/README.md

@@ -249,7 +249,8 @@ You can set message headers by passing a `headers:` option with a Hash of header
 ### Standalone Producer
 
 Racecar provides a standalone producer to publish messages to Kafka directly from your Rails application:
-```
+
+```ruby
 # app/controllers/comments_controller.rb
 class CommentsController < ApplicationController
   def create
@@ -263,7 +264,8 @@ end
 ```
 
 The above example will block the server process until the message has been delivered. If you want deliveries to happen in the background in order to free up your server processes more quickly, call #deliver_async instead:
-```
+
+```ruby
 # app/controllers/comments_controller.rb
 class CommentsController < ApplicationController
   def show
@@ -284,9 +286,9 @@ end
 ```
 In addition to improving response time, delivering messages asynchronously also protects your application against Kafka availability issues -- if messages cannot be delivered, they'll be buffered for later and retried automatically.
 
-A third method is to produce messages first (without delivering the messages to Kafka yet), and deliver them synchronously later.
+A third method is to produce messages first (without delivering the messages to Kafka yet), and deliver them synchronously later:
 
-```
+```ruby
  # app/controllers/comments_controller.rb
  class CommentsController < ApplicationController
    def create
@@ -415,7 +417,7 @@ Racecar supports [Datadog](https://www.datadoghq.com/) monitoring integration. I
 - `datadog_namespace` – The namespace to use for Datadog metrics.
 - `datadog_tags` – Tags that should always be set on Datadog metrics.
 
-Furthermore, there's a [standard Datadog dashboard configution file](https://raw.githubusercontent.com/zendesk/racecar/master/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
+Furthermore, there's a [standard Datadog dashboard configuration file](https://raw.githubusercontent.com/zendesk/racecar/master/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
 
 #### Consumers Without Rails
 
@@ -474,6 +476,8 @@ With Foreman, you can easily run these processes locally by executing `foreman r
 
 If you run your applications in Kubernetes, use the following [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) spec as a starting point:
 
+##### Recreate Strategy
+
 ```yaml
 apiVersion: apps/v1
 kind: Deployment
@@ -481,8 +485,8 @@ metadata:
   name: my-racecar-deployment
   labels:
     app: my-racecar
-spec:
-  replicas: 3 # <-- this will give us three consumers in the group.
+spec
+  replicas: 4 # <-- this is a good value if you have a multliple of 4 partitions
   selector:
     matchLabels:
       app: my-racecar
@@ -504,9 +508,33 @@ spec:
               value: 5
 ```
 
-The important part is the `strategy.type` value, which tells Kubernetes how to upgrade from one version of your Deployment to another. Many services use so-called _rolling updates_, where some but not all containers are replaced with the new version. This is done so that, if the new version doesn't work, the old version is still there to serve most of the requests. For Kafka consumers, this doesn't work well. The reason is that every time a consumer joins or leaves a group, every other consumer in the group needs to stop and synchronize the list of partitions assigned to each group member. So if the group is updated in a rolling fashion, this synchronization would occur over and over again, causing undesirable double-processing of messages as consumers would start only to be synchronized shortly after.
+This configuration uses the recreate strategy which completely terminates all consumers before starting new ones.
+It's simple and easy to understand but can result in significant 'downtime' where no messages are processed.
+
+##### Rolling Updates and 'sticky-cooperative' Assignment
+
+A newer alternative is to use the consumer's "cooperative-sticky" assignment strategy which allows healthy consumers to keep processing their partitions while others are terminated.
+This can be combined with a restricted rolling update to minimize processing downtime.
+
+Add to your Racecar config:
+```ruby
+Racecar.configure do |c|
+  c.partition_assignment_strategy = "cooperative-sticky"
+end
+```
+
+Replace the Kubernetes deployment strategy with:
+```yaml
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+        maxSurge: 0 # <- Never boot an excess consumer
+        maxUnavailable: 1 # <- The deploy 'rolls' one consumer at a time
+```
+
+These two configurations should be deployed together.
 
-Instead, the `Recreate` update strategy should be used. It completely tears down the existing containers before starting all of the new containers simultaneously, allowing for a single synchronization stage and a much faster, more stable deployment update.
+While `maxSurge` should always be 0, `maxUnavailable` can be increased to reduce deployment times in exchange for longer pauses in message processing.
 
 #### Liveness Probe
 
@@ -661,7 +689,7 @@ In order to introspect the configuration of a consumer process, send it the `SIG
 
 ### Upgrading from v1 to v2
 
-In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency. In general, you should avoid rolling deploys for consumers groups, so it is likely the case that this will just work for you, but it's a good idea to check first. 
+In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency.
 
 ### Compression
 
@@ -679,12 +707,12 @@ The integration tests run against a Kafka instance that is not automatically sta
 
 ### Running RSpec within Docker
 
-There can be behavioural inconsistencies between running the specs on your machine, and in the CI pipeline. Due to this, there is now a Dockerfile included in the project, which is based on the CircleCI ruby 2.7.2 image. This could easily be extended with more Dockerfiles to cover different Ruby versions if desired. In order to run the specs via Docker:
+There can be behavioural inconsistencies between running the specs on your machine, and in the CI pipeline. Due to this, there is now a Dockerfile included in the project, which is based on the CircleCI ruby 2.7.8 image. This could easily be extended with more Dockerfiles to cover different Ruby versions if desired. In order to run the specs via Docker:
 
 - Uncomment the `tests` service from the docker-compose.yml
 - Bring up the stack with `docker-compose up -d`
-- Execute the entire suite with `docker-compose run --rm tests rspec`
-- Execute a single spec or directory with `docker-compose run --rm tests rspec spec/integration/consumer_spec.rb`
+- Execute the entire suite with `docker-compose run --rm tests bundle exec rspec`
+- Execute a single spec or directory with `docker-compose run --rm tests bundle exec rspec spec/integration/consumer_spec.rb`
 
 Please note - your code directory is mounted as a volume, so you can make code changes without needing to rebuild
 

data/lib/racecar/cli.rb

@@ -19,6 +19,7 @@ module Racecar
       @parser = build_parser
       @parser.parse!(args)
       @consumer_name = args.first or raise Racecar::Error, "no consumer specified"
+      @runner = nil
     end
 
     def run
@@ -65,10 +66,15 @@ module Racecar
       end
 
       processor = consumer_class.new
-      Racecar.run(processor)
+      @runner = Racecar.runner(processor)
+      @runner.run
       nil
     end
 
+    def stop
+      @runner.stop
+    end
+
     private
 
     attr_reader :consumer_name

data/lib/racecar/config.rb

@@ -6,6 +6,7 @@ require "king_konf"
 
 require "racecar/liveness_probe"
 require "racecar/instrumenter"
+require "racecar/rebalance_listener"
 
 module Racecar
   class Config < KingKonf::Config
@@ -28,6 +29,9 @@ module Racecar
     desc "The minimum number of messages in the local consumer queue"
     integer :min_message_queue_size, default: 2000
 
+    desc "Which partition assignment strategy to use, range, roundrobin or cooperative-sticky. -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"
+    string :partition_assignment_strategy, default: "range,roundrobin"
+
     desc "Kafka consumer configuration options, separated with '=' -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"
     list :consumer, default: []
 
@@ -227,6 +231,7 @@ module Racecar
     end
 
     def load_consumer_class(consumer_class)
+      self.consumer_class = consumer_class
       self.group_id = consumer_class.group_id || self.group_id
 
       self.group_id ||= [
@@ -243,6 +248,7 @@ module Racecar
       self.fetch_messages = consumer_class.fetch_messages || self.fetch_messages
       self.pidfile ||= "#{group_id}.pid"
     end
+    attr_accessor :consumer_class
 
     def on_error(&handler)
       @error_handler = handler

data/lib/racecar/consumer.rb

@@ -37,6 +37,10 @@ module Racecar
           subscriptions << Subscription.new(topic, start_from_beginning, max_bytes_per_partition, additional_config)
         end
       end
+
+      # Rebalance hooks for subclasses to override
+      def on_partitions_assigned(rebalance_event); end
+      def on_partitions_revoked(rebalance_event); end
     end
 
     def configure(producer:, consumer:, instrumenter: NullInstrumenter, config: Racecar.config)

data/lib/racecar/consumer_set.rb

@@ -15,6 +15,7 @@ module Racecar
       @previous_retries = 0
 
       @last_poll_read_nil_message = false
+      @paused_tpls = Hash.new { |h, k| h[k] = {} }
     end
 
     def poll(max_wait_time_ms = @config.max_wait_time_ms)
@@ -65,11 +66,17 @@ module Racecar
 
     def close
       each_subscribed(&:close)
+      @paused_tpls.clear
     end
 
     def current
       @consumers[@consumer_id_iterator.peek] ||= begin
-        consumer = Rdkafka::Config.new(rdkafka_config(current_subscription)).consumer
+        consumer_config = Rdkafka::Config.new(rdkafka_config(current_subscription))
+        listener = RebalanceListener.new(@config.consumer_class, @instrumenter)
+        consumer_config.consumer_rebalance_listener = listener
+        consumer = consumer_config.consumer
+        listener.rdkafka_consumer = consumer
+
         @instrumenter.instrument('join_group') do
           consumer.subscribe current_subscription.topic
         end
@@ -95,16 +102,25 @@ module Racecar
       consumer.pause(filtered_tpl)
       fake_msg = OpenStruct.new(topic: topic, partition: partition, offset: offset)
       consumer.seek(fake_msg)
+
+      @paused_tpls[topic][partition] = [consumer, filtered_tpl]
     end
 
     def resume(topic, partition)
       consumer, filtered_tpl = find_consumer_by(topic, partition)
+
+      if !consumer && @paused_tpls[topic][partition]
+        consumer, filtered_tpl = @paused_tpls[topic][partition]
+      end
+
       if !consumer
         @logger.info "Attempted to resume #{topic}/#{partition}, but we're not subscribed to it"
         return
       end
 
       consumer.resume(filtered_tpl)
+      @paused_tpls[topic].delete(partition)
+      @paused_tpls.delete(topic) if @paused_tpls[topic].empty?
     end
 
     alias :each :each_subscribed
@@ -215,6 +231,11 @@ module Racecar
     def rdkafka_config(subscription)
       # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       config = {
+        # Manually store offset after messages have been processed successfully
+        # to avoid marking failed messages as committed. The call just updates
+        # a value within librdkafka and is asynchronously written to proper
+        # storage through auto commits.
+        "enable.auto.offset.store" => false,
         "auto.commit.interval.ms" => @config.offset_commit_interval * 1000,
         "auto.offset.reset"       => subscription.start_from_beginning ? "earliest" : "largest",
         "bootstrap.servers"       => @config.brokers.join(","),
@@ -230,7 +251,8 @@ module Racecar
         "queued.min.messages"     => @config.min_message_queue_size,
         "session.timeout.ms"      => @config.session_timeout * 1000,
         "socket.timeout.ms"       => @config.socket_timeout * 1000,
-        "statistics.interval.ms"  => @config.statistics_interval_ms
+        "statistics.interval.ms"  => @config.statistics_interval_ms,
+        "partition.assignment.strategy" => @config.partition_assignment_strategy,
       }
       config.merge! @config.rdkafka_consumer
       config.merge! subscription.additional_config

data/lib/racecar/ctl.rb

@@ -36,8 +36,10 @@ module Racecar
       require "racecar/liveness_probe"
       parse_options!(args)
 
-      if ENV["RAILS_ENV"]
-        Racecar.config.load_file("config/racecar.yml", ENV["RAILS_ENV"])
+      RailsConfigFileLoader.load! unless config.without_rails?
+
+      if File.exist?("config/racecar.rb")
+        require "./config/racecar"
       end
 
       Racecar.config.liveness_probe.check_liveness_within_interval!
@@ -136,5 +138,9 @@ module Racecar
 
       parser.parse!(args)
     end
+
+    def config
+      Racecar.config
+    end
   end
 end

data/lib/racecar/daemon.rb

@@ -54,7 +54,7 @@ module Racecar
     end
 
     def pid
-      if File.exists?(pidfile)
+      if File.exist?(pidfile)
         File.read(pidfile).to_i
       else
         nil
@@ -89,7 +89,7 @@ module Racecar
       end
 
       at_exit do
-        File.delete(pidfile) if File.exists?(pidfile)
+        File.delete(pidfile) if File.exist?(pidfile)
       end
     rescue Errno::EEXIST
       check_pid

data/lib/racecar/datadog.rb

@@ -211,6 +211,10 @@ module Racecar
           topic: topic,
         }
 
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
         # This gets us the write rate.
         increment("producer.produce.messages", tags: tags.merge(topic: topic))
 
@@ -244,7 +248,15 @@ module Racecar
         # Number of messages ACK'd for the topic.
         increment("producer.ack.messages", tags: tags)
       end
-      
+
+      def produce_delivery_error(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+        }
+
+        increment("producer.produce.delivery.errors", tags: tags)
+      end
+
       def produce_async(event)
         client = event.payload.fetch(:client_id)
         topic = event.payload.fetch(:topic)
@@ -256,6 +268,10 @@ module Racecar
           topic: topic,
         }
 
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
         # This gets us the write rate.
         increment("producer.produce.messages", tags: tags.merge(topic: topic))
 
@@ -279,6 +295,11 @@ module Racecar
           topic: topic,
         }
 
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
+
         # This gets us the write rate.
         increment("producer.produce.messages", tags: tags.merge(topic: topic))
 

data/lib/racecar/delivery_callback.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Racecar
+  class DeliveryCallback
+    attr_reader :instrumenter
+
+    def initialize(instrumenter:)
+      @instrumenter = instrumenter
+    end
+
+    def call(delivery_report)
+      if delivery_report.error.to_i.zero?
+        payload = {
+          offset: delivery_report.offset,
+          partition: delivery_report.partition
+        }
+        instrumenter.instrument("acknowledged_message", payload)
+      else
+        payload = {
+          partition: delivery_report.partition,
+          exception: delivery_report.error
+        }
+        instrumenter.instrument("produce_delivery_error", payload)
+      end
+    end
+  end
+end

data/lib/racecar/parallel_runner.rb

@@ -37,6 +37,10 @@ module Racecar
       wait_for_exit
     end
 
+    def stop
+      terminate_workers
+    end
+
     private
 
     attr_accessor :workers

data/lib/racecar/producer.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "racecar/message_delivery_error"
+require "racecar/delivery_callback"
 
 at_exit do
   Racecar::Producer.shutdown!
@@ -42,19 +43,24 @@ module Racecar
           }
           producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
           producer_config.merge!(config.rdkafka_producer)
-          Rdkafka::Config.new(producer_config).producer
+          Rdkafka::Config.new(producer_config).producer.tap do |producer|
+            producer.delivery_callback = DeliveryCallback.new(instrumenter: @instrumenter)
+          end
         end
       end
     end
 
-
-    # fire and forget - you won't get any guarantees or feedback from 
+    # fire and forget - you won't get any guarantees or feedback from
     # Racecar on the status of the message and it won't halt execution
     # of the rest of your code.
     def produce_async(value:, topic:, **options)
       with_instrumentation(action: "produce_async", value: value, topic: topic, **options) do
-        handle = internal_producer.produce(payload: value, topic: topic, **options)
-        @delivery_handles << handle if @batching
+        begin
+          handle = internal_producer.produce(payload: value, topic: topic, **options)
+          @delivery_handles << handle if @batching
+        rescue Rdkafka::RdkafkaError => e
+          raise MessageDeliveryError.new(e, handle)
+        end
       end
 
       nil
@@ -63,8 +69,12 @@ module Racecar
     # synchronous message production - will wait until the delivery handle succeeds, fails or times out.
     def produce_sync(value:, topic:, **options)
       with_instrumentation(action: "produce_sync", value: value, topic: topic, **options) do
-        handle = internal_producer.produce(payload: value, topic: topic, **options)
-        deliver_with_error_handling(handle)
+        begin
+          handle = internal_producer.produce(payload: value, topic: topic, **options)
+          deliver_with_error_handling(handle)
+        rescue Rdkafka::RdkafkaError => e
+          raise MessageDeliveryError.new(e, handle)
+        end
       end
 
       nil
@@ -126,4 +136,4 @@ module Racecar
       end
     end
   end
-end
+end

data/lib/racecar/rebalance_listener.rb

@@ -0,0 +1,58 @@
+module Racecar
+  class RebalanceListener
+    def initialize(consumer_class, instrumenter)
+      @consumer_class = consumer_class
+      @instrumenter = instrumenter
+      @rdkafka_consumer = nil
+    end
+
+    attr_writer :rdkafka_consumer
+
+    attr_reader :consumer_class, :instrumenter, :rdkafka_consumer
+    private     :consumer_class, :instrumenter, :rdkafka_consumer
+
+    def on_partitions_assigned(rdkafka_topic_partition_list)
+      event = Event.new(rdkafka_consumer: rdkafka_consumer, rdkafka_topic_partition_list: rdkafka_topic_partition_list)
+
+      instrument("partitions_assigned", partitions: event.partition_numbers) do
+        consumer_class.on_partitions_assigned(event)
+      end
+    end
+
+    def on_partitions_revoked(rdkafka_topic_partition_list)
+      event = Event.new(rdkafka_consumer: rdkafka_consumer, rdkafka_topic_partition_list: rdkafka_topic_partition_list)
+
+      instrument("partitions_revoked", partitions: event.partition_numbers) do
+        consumer_class.on_partitions_revoked(event)
+      end
+    end
+
+    private
+
+    def instrument(event, payload, &block)
+      instrumenter.instrument(event, payload, &block)
+    end
+
+    class Event
+      def initialize(rdkafka_topic_partition_list:, rdkafka_consumer:)
+        @__rdkafka_topic_partition_list = rdkafka_topic_partition_list
+        @__rdkafka_consumer = rdkafka_consumer
+      end
+
+      def topic_name
+        __rdkafka_topic_partition_list.to_h.keys.first
+      end
+
+      def partition_numbers
+        __rdkafka_topic_partition_list.to_h.values.flatten.map(&:partition)
+      end
+
+      def empty?
+        __rdkafka_topic_partition_list.empty?
+      end
+
+      # API private and not guaranteed stable
+      attr_reader :__rdkafka_topic_partition_list, :__rdkafka_consumer
+    end
+  end
+end

data/lib/racecar/runner.rb

@@ -5,6 +5,7 @@ require "racecar/pause"
 require "racecar/message"
 require "racecar/message_delivery_error"
 require "racecar/erroneous_state_error"
+require "racecar/delivery_callback"
 
 module Racecar
   class Runner
@@ -95,7 +96,7 @@ module Racecar
       end
     ensure
       producer.close
-      Racecar::Datadog.close if Object.const_defined?("Racecar::Datadog")
+      Racecar::Datadog.close if config.datadog_enabled
       @instrumenter.instrument("shut_down", instrumentation_payload || {})
     end
 
@@ -130,18 +131,13 @@ module Racecar
 
     def consumer
       @consumer ||= begin
-        # Manually store offset after messages have been processed successfully
-        # to avoid marking failed messages as committed. The call just updates
-        # a value within librdkafka and is asynchronously written to proper
-        # storage through auto commits.
-        config.consumer << "enable.auto.offset.store=false"
         ConsumerSet.new(config, logger, @instrumenter)
       end
     end
 
     def producer
       @producer ||= Rdkafka::Config.new(producer_config).producer.tap do |producer|
-        producer.delivery_callback = delivery_callback
+        producer.delivery_callback = Racecar::DeliveryCallback.new(instrumenter: @instrumenter)
       end
     end
 
@@ -160,16 +156,6 @@ module Racecar
       producer_config
     end
 
-    def delivery_callback
-      ->(delivery_report) do
-        payload = {
-          offset: delivery_report.offset,
-          partition: delivery_report.partition
-        }
-        @instrumenter.instrument("acknowledged_message", payload)
-      end
-    end
-
     def install_signal_handlers
       # Stop the consumer on SIGINT, SIGQUIT or SIGTERM.
       trap("QUIT") { stop }
@@ -222,21 +208,21 @@ module Racecar
       }
 
       @instrumenter.instrument("start_process_batch", instrumentation_payload)
-      @instrumenter.instrument("process_batch", instrumentation_payload) do
-        with_pause(first.topic, first.partition, first.offset..last.offset) do |pause|
-          begin
+      with_pause(first.topic, first.partition, first.offset..last.offset) do |pause|
+        begin
+          @instrumenter.instrument("process_batch", instrumentation_payload) do
             racecar_messages = messages.map do |message|
               Racecar::Message.new(message, retries_count: pause.pauses_count)
             end
             processor.process_batch(racecar_messages)
             processor.deliver!
             consumer.store_offset(messages.last)
-          rescue => e
-            instrumentation_payload[:unrecoverable_delivery_error] = reset_producer_on_unrecoverable_delivery_errors(e)
-            instrumentation_payload[:retries_count] = pause.pauses_count
-            config.error_handler.call(e, instrumentation_payload)
-            raise e
           end
+        rescue => e
+          instrumentation_payload[:unrecoverable_delivery_error] = reset_producer_on_unrecoverable_delivery_errors(e)
+          instrumentation_payload[:retries_count] = pause.pauses_count
+          config.error_handler.call(e, instrumentation_payload)
+          raise e
         end
       end
     end

data/lib/racecar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Racecar
-  VERSION = "2.9.0.beta1"
+  VERSION = "2.10.0.beta1"
 end

data/lib/racecar.rb

@@ -66,12 +66,16 @@ module Racecar
   end
 
   def self.run(processor)
+    runner(processor).run
+  end
+
+  def self.runner(processor)
     runner = Runner.new(processor, config: config, logger: logger, instrumenter: config.instrumenter)
 
     if config.parallel_workers && config.parallel_workers > 1
-      ParallelRunner.new(runner: runner, config: config, logger: logger).run
+      ParallelRunner.new(runner: runner, config: config, logger: logger)
     else
-      runner.run
+      runner
     end
   end
 end

data/racecar.gemspec

@@ -23,13 +23,13 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.6'
 
   spec.add_runtime_dependency "king_konf", "~> 1.0.0"
-  spec.add_runtime_dependency "rdkafka",   "~> 0.12.0"
+  spec.add_runtime_dependency "rdkafka",   "~> 0.13.0"
 
   spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
-  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "rake", "> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 6.0.0"
-  spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
+  spec.add_development_dependency "activesupport"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.9.0.beta1
+  version: 2.10.0.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-03-22 00:00:00.000000000 Z
+date: 2023-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.13.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.13.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -60,7 +60,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -141,20 +141,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '0'
 description: 
 email:
 - dschierbeck@zendesk.com
@@ -200,6 +194,7 @@ files:
 - lib/racecar/ctl.rb
 - lib/racecar/daemon.rb
 - lib/racecar/datadog.rb
+- lib/racecar/delivery_callback.rb
 - lib/racecar/erroneous_state_error.rb
 - lib/racecar/heroku.rb
 - lib/racecar/instrumenter.rb
@@ -211,6 +206,7 @@ files:
 - lib/racecar/pause.rb
 - lib/racecar/producer.rb
 - lib/racecar/rails_config_file_loader.rb
+- lib/racecar/rebalance_listener.rb
 - lib/racecar/runner.rb
 - lib/racecar/version.rb
 - racecar.gemspec