racecar 2.9.0 → 2.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b394d31edcb9c83e562811ba748b6c6915388d0227a4e7425aa9aa13f64e3890
-  data.tar.gz: 1dfcd7046b6d932f716246d0685589c1635af115a4e126067c9775b13fe05464
+  metadata.gz: a4e6b0c4d52637bd8c6bb2c5e3d69013889053973195a5d73ac5d5645047f985
+  data.tar.gz: 23425d552932665cafa03cf245d3d4ef879e053e9e9c60d1eb172455f405f3f3
 SHA512:
-  metadata.gz: 59043be21e411e680c11815b583236556413a539f5aa9508a460eefe82cee6199f56a30b9fa07a3f206886d5dc811b210cea6130813a7073a68fe6612343ca0d
-  data.tar.gz: d6692b9bb7cdc27efe5272a10fa1d9061920084a7b517699e5133f5e31e9fd50415e72ee40550e07e5a2fed028aeabb741422fef47372c7a53e0a96bdfc1b090
+  metadata.gz: 62d52be41cb3821a5d6534fea43deea8d02dd8a8f03d3449249be0fcdcbf5c842cbb2be8aac7065b97fb515b16370ab93f114ab44dd85dce6a22091d9ac04de4
+  data.tar.gz: 006efc7649ddb4257a80e1f2400a62e9e9cba0fa5573d3947f47293d4d77b9b701d66ebf6d465cf91fbf209771e5699ddc9cd7df238ecddf7f54ffbab4eb0c15

data/.github/workflows/ci.yml

@@ -11,17 +11,18 @@ jobs:
     runs-on: ubuntu-latest
 
     strategy:
+      fail-fast: false
       matrix:
         ruby-version:
-          - "2.7"
-          - "3.0"
           - "3.1"
           - "3.2"
+          - "3.3"
+          - "3.4"
 
     steps:
-    - uses: zendesk/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Ruby
-      uses: zendesk/setup-ruby@v1
+      uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true
@@ -30,15 +31,23 @@ jobs:
 
   integration-specs:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - "3.1"
+          - "3.2"
+          - "3.3"
+          - "3.4"
     steps:
-    - uses: zendesk/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Ruby
-      uses: zendesk/setup-ruby@v1
+      uses: ruby/setup-ruby@v1
       with:
-        ruby-version: "2.7"
+        ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true
     - name: Bring up docker-compose stack
-      run: docker-compose up -d
+      run: docker compose up -d
     - name: Build and test with RSpec
       env:
         RACECAR_BROKERS: localhost:9092

data/.rspec

@@ -1,3 +1,4 @@
 --format documentation
 --require spec_helper
 --color
+--order random

data/CHANGELOG.md

@@ -2,7 +2,31 @@
 
 ## Unreleased
 
-* Test with Ruby 3.2
+## 2.12.0
+
+* Add tests against Ruby 3.4
+* Drop support for Ruby 3.0
+* Allow rdkafka gem versions newer than 0.15.0
+
+## 2.11.0
+
+* Allow the liveness probe command to skip loading config files
+* Add unix domain socket support for Datadog StatsD metrics
+* Bump minimum rdkafka gem version to 0.15.0
+* Bump minimum Ruby version to 3.0
+* Configurable strategy for consuming multiple topics
+
+## 2.10.0
+
+* Bump rdkafka gem version to 0.13.0
+* Support cooperative-sticky
+* Instrument produce delivery errors
+* Resolve Rails 7.1 logger incompatibility
+* Don't load rails env for liveness probe
+* Fix config load for liveness probe
+* Send exceptions to `process_batch` instrumenter
+* Docker test fixes
+* Test in CI with Ruby 3.2
 
 ## v2.9.0, v2.9.0.beta1
 
@@ -12,7 +36,7 @@
 * Add built-in liveness probe for Kubernetes deployments.
 
 ## v2.8.2
-* Handles ErroneousStateError, in previous versions the consumer would do several unecessary group leave/joins. The log level is also changed to WARN instead of ERROR. ([#295](https://github.com/zendesk/racecar/pull/295))
+* Handles ErroneousStateError, in previous versions the consumer would do several unnecessary group leave/joins. The log level is also changed to WARN instead of ERROR. ([#295](https://github.com/zendesk/racecar/pull/295))
 
 ## v2.8.1
 * Adds new ErroneousStateError to racecar in order to give more information on this new possible exception.

data/Dockerfile

@@ -1,4 +1,4 @@
-FROM circleci/ruby:2.7.2
+FROM cimg/ruby:2.7.8
 
 RUN sudo apt-get update
 RUN sudo apt-get install docker

data/Gemfile

@@ -5,6 +5,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in racecar.gemspec
 gemspec
 
-# We actually support version 7.x (see gemspec); this extra restriction is added just for running the test suite also
-# on Ruby 2.6, which activesupport 7.0 does not support.
-gem 'activesupport', '~> 6.1.0'
+gem 'activesupport', '~> 7.2.0'

data/Gemfile.lock

@@ -1,31 +1,43 @@
 PATH
   remote: .
   specs:
-    racecar (2.9.0.beta1)
+    racecar (2.12.0)
       king_konf (~> 1.0.0)
-      rdkafka (~> 0.12.0)
+      rdkafka (>= 0.15.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.7.3)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+    activesupport (7.2.2.1)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
     byebug (11.1.3)
     coderay (1.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.3.4)
+    connection_pool (2.4.1)
     diff-lcs (1.5.0)
     dogstatsd-ruby (5.5.0)
-    ffi (1.15.5)
+    drb (2.2.1)
+    ffi (1.16.3)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     king_konf (1.0.1)
+    logger (1.6.4)
     method_source (1.0.0)
-    mini_portile2 (2.8.1)
-    minitest (5.18.0)
+    mini_portile2 (2.8.5)
+    minitest (5.25.4)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -33,7 +45,7 @@ GEM
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
     rake (13.0.6)
-    rdkafka (0.12.0)
+    rdkafka (0.18.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
@@ -50,16 +62,16 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
+    securerandom (0.4.1)
     timecop (0.9.6)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.7)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport (~> 6.1.0)
+  activesupport (~> 7.2.0)
   bundler (>= 1.13, < 3)
   dogstatsd-ruby (>= 4.0.0, < 6.0.0)
   pry-byebug
@@ -69,4 +81,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   2.4.9
+   2.6.2

data/README.md

@@ -414,6 +414,7 @@ Racecar supports [Datadog](https://www.datadoghq.com/) monitoring integration. I
 - `datadog_enabled` – Whether Datadog monitoring is enabled (defaults to `false`).
 - `datadog_host` – The host running the Datadog agent.
 - `datadog_port` – The port of the Datadog agent.
+- `datadog_socket_path` – The unix domain socket of the Datadog agent (when set takes precedence over host/port).
 - `datadog_namespace` – The namespace to use for Datadog metrics.
 - `datadog_tags` – Tags that should always be set on Datadog metrics.
 
@@ -476,6 +477,8 @@ With Foreman, you can easily run these processes locally by executing `foreman r
 
 If you run your applications in Kubernetes, use the following [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) spec as a starting point:
 
+##### Recreate Strategy
+
 ```yaml
 apiVersion: apps/v1
 kind: Deployment
@@ -483,8 +486,8 @@ metadata:
   name: my-racecar-deployment
   labels:
     app: my-racecar
-spec:
-  replicas: 3 # <-- this will give us three consumers in the group.
+spec
+  replicas: 4 # <-- this is a good value if you have a multliple of 4 partitions
   selector:
     matchLabels:
       app: my-racecar
@@ -506,24 +509,47 @@ spec:
               value: 5
 ```
 
-The important part is the `strategy.type` value, which tells Kubernetes how to upgrade from one version of your Deployment to another. Many services use so-called _rolling updates_, where some but not all containers are replaced with the new version. This is done so that, if the new version doesn't work, the old version is still there to serve most of the requests. For Kafka consumers, this doesn't work well. The reason is that every time a consumer joins or leaves a group, every other consumer in the group needs to stop and synchronize the list of partitions assigned to each group member. So if the group is updated in a rolling fashion, this synchronization would occur over and over again, causing undesirable double-processing of messages as consumers would start only to be synchronized shortly after.
+This configuration uses the recreate strategy which completely terminates all consumers before starting new ones.
+It's simple and easy to understand but can result in significant 'downtime' where no messages are processed.
+
+##### Rolling Updates and 'sticky-cooperative' Assignment
 
-Instead, the `Recreate` update strategy should be used. It completely tears down the existing containers before starting all of the new containers simultaneously, allowing for a single synchronization stage and a much faster, more stable deployment update.
+A newer alternative is to use the consumer's "cooperative-sticky" assignment strategy which allows healthy consumers to keep processing their partitions while others are terminated.
+This can be combined with a restricted rolling update to minimize processing downtime.
+
+Add to your Racecar config:
+```ruby
+Racecar.configure do |c|
+  c.partition_assignment_strategy = "cooperative-sticky"
+end
+```
+
+Replace the Kubernetes deployment strategy with:
+```yaml
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+        maxSurge: 0 # <- Never boot an excess consumer
+        maxUnavailable: 1 # <- The deploy 'rolls' one consumer at a time
+```
+
+These two configurations should be deployed together.
+
+While `maxSurge` should always be 0, `maxUnavailable` can be increased to reduce deployment times in exchange for longer pauses in message processing.
 
 #### Liveness Probe
 
 Racecar comes with a built-in liveness probe, primarily for use with Kubernetes, but useful for any deployment environment where you can periodically run a process to check the health of your consumer.
 
 To use this feature:
-- set the `liveness_probe_enabled` config option to true.
-- configure your Kubernetes deployment to run `$ racecarctl liveness_probe`
+1. Set the `liveness_probe_enabled` config option to true.
+2. Configure your Kubernetes deployment liveness probe to run this command `$ racecarctl liveness_probe`
 
-
-When enabled (see config) Racecar will touch the file at `liveness_probe_file_path` each time it finishes polling Kafka and processing the messages in the batch (if any).
+When enabled (see config) Racecar will touch the file at the specified path each time it finishes polling Kafka and processing the messages in the batch (if any).
 
 The modified time of this file can be observed to determine when the consumer last exhibited 'liveness'.
 
-Running `racecarctl liveness_probe` will return a successful exit status if the last 'liveness' event happened within an acceptable time, `liveness_probe_max_interval`.
+Running `racecarctl liveness_probe` will return a successful exit status if the last 'liveness' event happened within an acceptable time, which you can set as `liveness_probe_max_interval`.
 
 `liveness_probe_max_interval` should be long enough to account for both the Kafka polling time of `max_wait_time` and the processing time of a full message batch.
 
@@ -531,9 +557,15 @@ On receiving `SIGTERM`, Racecar will gracefully shut down and delete this file,
 
 You may wish to tolerate more than one failed probe run to accommodate for environmental variance and clock changes.
 
-See the [Configuration section](https://github.com/zendesk/racecar#configuration) for the various ways the liveness probe can be configured, environment variables being one option.
+The [Configuration section](https://github.com/zendesk/racecar#configuration) for the various ways the liveness probe can be configured. (We recommend environment variables).
+
+##### Slow racecar.rb / racecar.yml? Skip config files!
 
-Here is an example Kubernetes liveness probe configuration:
+If your config files need to do something expensive, such as load Rails, you can enable `RACECAR_LIVENESS_PROBE_SKIP_CONFIG_FILES`. The liveness probe command will then skip loading your configuration and execute quickly.
+
+Most other configuration values can be set via the environment, we recommend you do this for liveness probe settings.
+
+##### Example Kubernetes Configuration
 
 ```yaml
 apiVersion: apps/v1
@@ -549,8 +581,15 @@ spec:
         - SomeConsumer
 
         env:
+        # Skip config loading to run fast, only the following values are needed
+        - name: RACECAR_LIVENESS_PROBE_SKIP_CONFIG_FILES
+          value: "true"
         - name: RACECAR_LIVENESS_PROBE_ENABLED
           value: "true"
+        - name: RACECAR_LIVENESS_PROBE_FILE_PATH
+          value: "/tmp/racecar-liveness"
+        - name: RACECAR_LIVENESS_PROBE_MAX_INTERVAL
+          value: "5"
 
         livenessProbe:
           exec:
@@ -558,8 +597,8 @@ spec:
             - racecarctl
             - liveness_probe
 
-          # Allow up to 10 consecutive failures before terminating Pod:
-          failureThreshold: 10
+          # Allow up to 3 consecutive failures before terminating Pod:
+          failureThreshold: 3
 
           # Wait 30 seconds before starting the probes:
           initialDelaySeconds: 30
@@ -663,13 +702,13 @@ In order to introspect the configuration of a consumer process, send it the `SIG
 
 ### Upgrading from v1 to v2
 
-In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency. In general, you should avoid rolling deploys for consumers groups, so it is likely the case that this will just work for you, but it's a good idea to check first. 
+In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency.
 
 ### Compression
 
 Racecar v2 requires a C library (zlib) to compress the messages before producing to the topic. If not already installed on you consumer docker container, please install using following command in Dockerfile of consumer
 
-``` 
+```
 apt-get update && apt-get install -y libzstd-dev
 ```
 
@@ -677,16 +716,16 @@ apt-get update && apt-get install -y libzstd-dev
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-The integration tests run against a Kafka instance that is not automatically started from within `rspec`. You can set one up using the provided `docker-compose.yml` by running `docker-compose up`.
+The integration tests run against a Kafka instance that is not automatically started from within `rspec`. You can set one up using the provided `docker-compose.yml` by running `docker compose up`.
 
 ### Running RSpec within Docker
 
-There can be behavioural inconsistencies between running the specs on your machine, and in the CI pipeline. Due to this, there is now a Dockerfile included in the project, which is based on the CircleCI ruby 2.7.2 image. This could easily be extended with more Dockerfiles to cover different Ruby versions if desired. In order to run the specs via Docker:
+There can be behavioural inconsistencies between running the specs on your machine, and in the CI pipeline. Due to this, there is now a Dockerfile included in the project, which is based on the CircleCI ruby 2.7.8 image. This could easily be extended with more Dockerfiles to cover different Ruby versions if desired. In order to run the specs via Docker:
 
 - Uncomment the `tests` service from the docker-compose.yml
-- Bring up the stack with `docker-compose up -d`
-- Execute the entire suite with `docker-compose run --rm tests rspec`
-- Execute a single spec or directory with `docker-compose run --rm tests rspec spec/integration/consumer_spec.rb`
+- Bring up the stack with `docker compose up -d`
+- Execute the entire suite with `docker compose run --rm tests bundle exec rspec`
+- Execute a single spec or directory with `docker compose run --rm tests bundle exec rspec spec/integration/consumer_spec.rb`
 
 Please note - your code directory is mounted as a volume, so you can make code changes without needing to rebuild
 

data/docker-compose.yml

@@ -1,8 +1,6 @@
-version: '2.1'
-
 services:
   zookeeper:
-    image: confluentinc/cp-zookeeper:5.5.1
+    image: confluentinc/cp-zookeeper:7.8.1
     ports:
       - "2181:2181"
     environment:
@@ -13,7 +11,7 @@ services:
       test: echo ruok | nc 127.0.0.1 2181 | grep imok
 
   broker:
-    image: confluentinc/cp-kafka:5.5.1
+    image: confluentinc/cp-kafka:7.8.1
     depends_on:
       - zookeeper
     ports:
@@ -57,7 +55,7 @@ services:
   #     RACECAR_BROKERS: broker:29092
   #     DOCKER_SUDO: 'true'
   #   # When bringing up the stack, we just let the container exit. For running the
-  #   # specs, we'll use commands like `docker-compose run tests rspec`
+  #   # specs, we'll use commands like `docker compose run tests rspec`
   #   command: ["echo", "ready"]
   #   volumes:
   #     # The line below allows us to run docker commands from the container itself

data/lib/racecar/cli.rb

@@ -157,10 +157,11 @@ module Racecar
       require_relative './datadog'
 
       Datadog.configure do |datadog|
-        datadog.host      = config.datadog_host unless config.datadog_host.nil?
-        datadog.port      = config.datadog_port unless config.datadog_port.nil?
-        datadog.namespace = config.datadog_namespace unless config.datadog_namespace.nil?
-        datadog.tags      = config.datadog_tags unless config.datadog_tags.nil?
+        datadog.host        = config.datadog_host unless config.datadog_host.nil?
+        datadog.port        = config.datadog_port unless config.datadog_port.nil?
+        datadog.socket_path = config.datadog_socket_path unless config.datadog_socket_path.nil?
+        datadog.namespace   = config.datadog_namespace unless config.datadog_namespace.nil?
+        datadog.tags        = config.datadog_tags unless config.datadog_tags.nil?
       end
     end
   end

data/lib/racecar/config.rb

@@ -29,6 +29,9 @@ module Racecar
     desc "The minimum number of messages in the local consumer queue"
     integer :min_message_queue_size, default: 2000
 
+    desc "Which partition assignment strategy to use, range, roundrobin or cooperative-sticky. -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"
+    string :partition_assignment_strategy, default: "range,roundrobin"
+
     desc "Kafka consumer configuration options, separated with '=' -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"
     list :consumer, default: []
 
@@ -155,6 +158,9 @@ module Racecar
     desc "The port of the Datadog agent"
     integer :datadog_port
 
+    desc "The unix domain socket of the Datadog agent (when set takes precedence over host/port)"
+    string :datadog_socket_path
+
     desc "The namespace to use for Datadog metrics"
     string :datadog_namespace
 
@@ -182,6 +188,12 @@ module Racecar
     desc "Used only by the liveness probe: Max time (in seconds) between liveness events before the process is considered not healthy"
     integer :liveness_probe_max_interval, default: 5
 
+    desc "Allows the liveness probe command to skip loading config files. When enabled, configure liveness probe values via environmental variables. Defaults still apply. Only applies to the liveness probe command."
+    boolean :liveness_probe_skip_config_files, default: false
+
+    desc "Strategy for switching topics when there are multiple subscriptions. `exhaust-topic` will only switch when the consumer poll returns no messages. `round-robin` will switch after each poll regardless.\nWarning: `round-robin` will be the default in Racecar 3.x"
+    string :multi_subscription_strategy, allowed_values: %w(round-robin exhaust-topic), default: "exhaust-topic"
+
     # The error handler must be set directly on the object.
     attr_reader :error_handler
 
@@ -296,10 +308,6 @@ module Racecar
       )
     end
 
-    def rebalance_listener
-      RebalanceListener.new(self)
-    end
-
     private
 
     def rdkafka_security_config

data/lib/racecar/consumer.rb

@@ -37,6 +37,10 @@ module Racecar
           subscriptions << Subscription.new(topic, start_from_beginning, max_bytes_per_partition, additional_config)
         end
       end
+
+      # Rebalance hooks for subclasses to override
+      def on_partitions_assigned(rebalance_event); end
+      def on_partitions_revoked(rebalance_event); end
     end
 
     def configure(producer:, consumer:, instrumenter: NullInstrumenter, config: Racecar.config)
@@ -64,16 +68,16 @@ module Racecar
         @instrumenter.instrument('deliver_messages', instrumentation_payload) do
           @delivery_handles.each do |handle|
             begin
-              # rdkafka-ruby checks every wait_timeout seconds if the message was
-              # successfully delivered, up to max_wait_timeout seconds before raising
-              # Rdkafka::AbstractHandle::WaitTimeoutError. librdkafka will (re)try to
-              # deliver all messages in the background, until "config.message_timeout"
+              # rdkafka-ruby checks with exponential backoff starting at 0 seconds wait
+              # if the message was successfully delivered, up to max_wait_timeout seconds
+              # before raising Rdkafka::AbstractHandle::WaitTimeoutError. librdkafka will 
+              # (re)try to deliver all messages in the background, until "config.message_timeout"
               # (message.timeout.ms) is exceeded. Phrased differently, rdkafka-ruby's
               # WaitTimeoutError is just informative.
               # The raising can be avoided if max_wait_timeout below is greater than
               # config.message_timeout, but config is not available here (without
               # changing the interface).
-              handle.wait(max_wait_timeout: 60, wait_timeout: 0.1)
+              handle.wait(max_wait_timeout: 60)
             rescue Rdkafka::AbstractHandle::WaitTimeoutError => e
               partition = MessageDeliveryError.partition_from_delivery_handle(handle)
               # ideally we could use the logger passed to the Runner, but it is not

data/lib/racecar/consumer_set.rb

@@ -15,6 +15,7 @@ module Racecar
       @previous_retries = 0
 
       @last_poll_read_nil_message = false
+      @paused_tpls = Hash.new { |h, k| h[k] = {} }
     end
 
     def poll(max_wait_time_ms = @config.max_wait_time_ms)
@@ -65,14 +66,17 @@ module Racecar
 
     def close
       each_subscribed(&:close)
+      @paused_tpls.clear
     end
 
     def current
       @consumers[@consumer_id_iterator.peek] ||= begin
         consumer_config = Rdkafka::Config.new(rdkafka_config(current_subscription))
-        consumer_config.consumer_rebalance_listener = @config.rebalance_listener
-
+        listener = RebalanceListener.new(@config.consumer_class, @instrumenter)
+        consumer_config.consumer_rebalance_listener = listener
         consumer = consumer_config.consumer
+        listener.rdkafka_consumer = consumer
+
         @instrumenter.instrument('join_group') do
           consumer.subscribe current_subscription.topic
         end
@@ -98,16 +102,25 @@ module Racecar
       consumer.pause(filtered_tpl)
       fake_msg = OpenStruct.new(topic: topic, partition: partition, offset: offset)
       consumer.seek(fake_msg)
+
+      @paused_tpls[topic][partition] = [consumer, filtered_tpl]
     end
 
     def resume(topic, partition)
       consumer, filtered_tpl = find_consumer_by(topic, partition)
+
+      if !consumer && @paused_tpls[topic][partition]
+        consumer, filtered_tpl = @paused_tpls[topic][partition]
+      end
+
       if !consumer
         @logger.info "Attempted to resume #{topic}/#{partition}, but we're not subscribed to it"
         return
       end
 
       consumer.resume(filtered_tpl)
+      @paused_tpls[topic].delete(partition)
+      @paused_tpls.delete(topic) if @paused_tpls[topic].empty?
     end
 
     alias :each :each_subscribed
@@ -162,6 +175,7 @@ module Racecar
 
     # polls a message for the current consumer, handling any API edge cases.
     def poll_current_consumer(max_wait_time_ms)
+      @last_poll_read_nil_message = false
       msg = current.poll(max_wait_time_ms)
     rescue Rdkafka::RdkafkaError => e
       case e.code
@@ -199,9 +213,14 @@ module Racecar
     end
 
     def maybe_select_next_consumer
-      return unless @last_poll_read_nil_message
-      @last_poll_read_nil_message = false
-      select_next_consumer
+      case @config.multi_subscription_strategy
+      when "round-robin"
+        select_next_consumer
+      else # "exhaust-topic"
+        if @last_poll_read_nil_message
+          select_next_consumer
+        end
+      end
     end
 
     def select_next_consumer
@@ -218,6 +237,11 @@ module Racecar
     def rdkafka_config(subscription)
       # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       config = {
+        # Manually store offset after messages have been processed successfully
+        # to avoid marking failed messages as committed. The call just updates
+        # a value within librdkafka and is asynchronously written to proper
+        # storage through auto commits.
+        "enable.auto.offset.store" => false,
         "auto.commit.interval.ms" => @config.offset_commit_interval * 1000,
         "auto.offset.reset"       => subscription.start_from_beginning ? "earliest" : "largest",
         "bootstrap.servers"       => @config.brokers.join(","),
@@ -233,7 +257,8 @@ module Racecar
         "queued.min.messages"     => @config.min_message_queue_size,
         "session.timeout.ms"      => @config.session_timeout * 1000,
         "socket.timeout.ms"       => @config.socket_timeout * 1000,
-        "statistics.interval.ms"  => @config.statistics_interval_ms
+        "statistics.interval.ms"  => @config.statistics_interval_ms,
+        "partition.assignment.strategy" => @config.partition_assignment_strategy,
       }
       config.merge! @config.rdkafka_consumer
       config.merge! subscription.additional_config

data/lib/racecar/ctl.rb

@@ -36,8 +36,14 @@ module Racecar
       require "racecar/liveness_probe"
       parse_options!(args)
 
-      if ENV["RAILS_ENV"]
-        Racecar.config.load_file("config/racecar.yml", ENV["RAILS_ENV"])
+      unless config.liveness_probe_skip_config_files
+        if File.exist?("config/racecar.rb")
+          require "./config/racecar"
+        end
+
+        if ENV["RAILS_ENV"] && File.exist?("config/racecar.yml")
+          Racecar.config.load_file("config/racecar.yml", ENV["RAILS_ENV"])
+        end
       end
 
       Racecar.config.liveness_probe.check_liveness_within_interval!
@@ -136,5 +142,9 @@ module Racecar
 
       parser.parse!(args)
     end
+
+    def config
+      Racecar.config
+    end
   end
 end

data/lib/racecar/datadog.rb

@@ -19,7 +19,11 @@ module Racecar
       end
 
       def statsd
-        @statsd ||= ::Datadog::Statsd.new(host, port, namespace: namespace, tags: tags)
+        @statsd ||= if socket_path
+          ::Datadog::Statsd.new(socket_path: socket_path, namespace: namespace, tags: tags)
+        else
+          ::Datadog::Statsd.new(host, port, namespace: namespace, tags: tags)
+        end
       end
 
       def statsd=(statsd)
@@ -45,6 +49,15 @@ module Racecar
         clear
       end
 
+      def socket_path
+        @socket_path
+      end
+
+      def socket_path=(socket_path)
+        @socket_path = socket_path
+        clear
+      end
+
       def namespace
         @namespace ||= STATSD_NAMESPACE
       end

data/lib/racecar/producer.rb

@@ -40,6 +40,7 @@ module Racecar
             "client.id"              => config.client_id,
             "statistics.interval.ms" => config.statistics_interval_ms,
             "message.timeout.ms"     => config.message_timeout * 1000,
+            "partitioner"            => config.partitioner.to_s
           }
           producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
           producer_config.merge!(config.rdkafka_producer)

data/lib/racecar/rails_config_file_loader.rb

@@ -26,7 +26,11 @@ module Racecar
           console = ActiveSupport::Logger.new($stdout)
           console.formatter = Rails.logger.formatter
           console.level = Rails.logger.level
-          Rails.logger.extend(ActiveSupport::Logger.broadcast(console))
+          if ::Rails::VERSION::STRING < "7.1"
+            Rails.logger.extend(ActiveSupport::Logger.broadcast(console))
+          else
+            Rails.logger = ActiveSupport::BroadcastLogger.new(Rails.logger, console)
+          end
         end
       end
     end

data/lib/racecar/rebalance_listener.rb

@@ -1,22 +1,58 @@
 module Racecar
   class RebalanceListener
-    def initialize(config)
-      @config = config
-      @consumer_class = config.consumer_class
+    def initialize(consumer_class, instrumenter)
+      @consumer_class = consumer_class
+      @instrumenter = instrumenter
+      @rdkafka_consumer = nil
     end
 
-    attr_reader :config, :consumer_class
+    attr_writer :rdkafka_consumer
 
-    def on_partitions_assigned(_consumer, topic_partition_list)
-      consumer_class.respond_to?(:on_partitions_assigned) &&
-        consumer_class.on_partitions_assigned(topic_partition_list.to_h)
-    rescue
+    attr_reader :consumer_class, :instrumenter, :rdkafka_consumer
+    private     :consumer_class, :instrumenter, :rdkafka_consumer
+
+    def on_partitions_assigned(rdkafka_topic_partition_list)
+      event = Event.new(rdkafka_consumer: rdkafka_consumer, rdkafka_topic_partition_list: rdkafka_topic_partition_list)
+
+      instrument("partitions_assigned", partitions: event.partition_numbers) do
+        consumer_class.on_partitions_assigned(event)
+      end
+    end
+
+    def on_partitions_revoked(rdkafka_topic_partition_list)
+      event = Event.new(rdkafka_consumer: rdkafka_consumer, rdkafka_topic_partition_list: rdkafka_topic_partition_list)
+
+      instrument("partitions_revoked", partitions: event.partition_numbers) do
+        consumer_class.on_partitions_revoked(event)
+      end
+    end
+
+    private
+
+    def instrument(event, payload, &block)
+      instrumenter.instrument(event, payload, &block)
     end
 
-    def on_partitions_revoked(_consumer, topic_partition_list)
-      consumer_class.respond_to?(:on_partitions_revoked) &&
-        consumer_class.on_partitions_revoked(topic_partition_list.to_h)
-    rescue
+    class Event
+      def initialize(rdkafka_topic_partition_list:, rdkafka_consumer:)
+        @__rdkafka_topic_partition_list = rdkafka_topic_partition_list
+        @__rdkafka_consumer = rdkafka_consumer
+      end
+
+      def topic_name
+        __rdkafka_topic_partition_list.to_h.keys.first
+      end
+
+      def partition_numbers
+        __rdkafka_topic_partition_list.to_h.values.flatten.map(&:partition)
+      end
+
+      def empty?
+        __rdkafka_topic_partition_list.empty?
+      end
+
+      # API private and not guaranteed stable
+      attr_reader :__rdkafka_topic_partition_list, :__rdkafka_consumer
     end
   end
 end

data/lib/racecar/runner.rb

@@ -96,7 +96,7 @@ module Racecar
       end
     ensure
       producer.close
-      Racecar::Datadog.close if Object.const_defined?("Racecar::Datadog")
+      Racecar::Datadog.close if config.datadog_enabled
       @instrumenter.instrument("shut_down", instrumentation_payload || {})
     end
 
@@ -131,11 +131,6 @@ module Racecar
 
     def consumer
       @consumer ||= begin
-        # Manually store offset after messages have been processed successfully
-        # to avoid marking failed messages as committed. The call just updates
-        # a value within librdkafka and is asynchronously written to proper
-        # storage through auto commits.
-        config.consumer << "enable.auto.offset.store=false"
         ConsumerSet.new(config, logger, @instrumenter)
       end
     end
@@ -213,21 +208,21 @@ module Racecar
       }
 
       @instrumenter.instrument("start_process_batch", instrumentation_payload)
-      @instrumenter.instrument("process_batch", instrumentation_payload) do
-        with_pause(first.topic, first.partition, first.offset..last.offset) do |pause|
-          begin
+      with_pause(first.topic, first.partition, first.offset..last.offset) do |pause|
+        begin
+          @instrumenter.instrument("process_batch", instrumentation_payload) do
             racecar_messages = messages.map do |message|
               Racecar::Message.new(message, retries_count: pause.pauses_count)
             end
             processor.process_batch(racecar_messages)
             processor.deliver!
             consumer.store_offset(messages.last)
-          rescue => e
-            instrumentation_payload[:unrecoverable_delivery_error] = reset_producer_on_unrecoverable_delivery_errors(e)
-            instrumentation_payload[:retries_count] = pause.pauses_count
-            config.error_handler.call(e, instrumentation_payload)
-            raise e
           end
+        rescue => e
+          instrumentation_payload[:unrecoverable_delivery_error] = reset_producer_on_unrecoverable_delivery_errors(e)
+          instrumentation_payload[:retries_count] = pause.pauses_count
+          config.error_handler.call(e, instrumentation_payload)
+          raise e
         end
       end
     end

data/lib/racecar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Racecar
-  VERSION = "2.9.0"
+  VERSION = "2.12.0"
 end

data/racecar.gemspec

@@ -20,10 +20,10 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = '>= 2.6'
+  spec.required_ruby_version = '>= 3.1'
 
   spec.add_runtime_dependency "king_konf", "~> 1.0.0"
-  spec.add_runtime_dependency "rdkafka",   "~> 0.12.0"
+  spec.add_runtime_dependency "rdkafka",   ">= 0.15.0"
 
   spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
   spec.add_development_dependency "pry-byebug"

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.9.0
+  version: 2.12.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 - Benjamin Quorning
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-09-25 00:00:00.000000000 Z
+date: 2025-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -29,16 +29,16 @@ dependencies:
   name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.15.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.15.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -149,7 +149,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - dschierbeck@zendesk.com
 - bquorning@zendesk.com
@@ -214,7 +214,7 @@ homepage: https://github.com/zendesk/racecar
 licenses:
 - Apache License Version 2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -222,15 +222,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: A framework for running Kafka consumers
 test_files: []