racecar 2.11.0 → 3.0.0.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7bdff11cf4a069df69e0a54651daa9c01ba29f1dc6182ec4743da1516ba32d54
-  data.tar.gz: 76f26192c1fdd82c95f3f0f23b4c64f18c10be1febceb585b3689136544ecb2b
+  metadata.gz: 8d48006d6ad7a9f1cb57902edbd8a2785f3e05c629a71c3345238a0841abeda6
+  data.tar.gz: 06b557c5757e1c4bda2d1d6cebd2e679ab27053bc02f58636911f8df8d67f217
 SHA512:
-  metadata.gz: 10656d95fba4b15e16ef742e6483f3e26de66dc192a50de2f5023134505695f10884bd8cbf5679cf5cc5cc18f76038ec5699fe68813c22ea7e73b8b3387989bf
-  data.tar.gz: 0ea9a87038a2dd66275910cb9e651d648f8cddf5a311d29aed1af0c4ab3a395ae7812baaa9d673fc3954e18f684f03dbf40e4b202f5c58c0997202426bda777e
+  metadata.gz: 9e6cd591d16f776a84140105fd70eb565c70c6001ac03c5f61ea0ef45b40a8fd29e656a8da4b445894c87d9bb6002376364d8de896af481847222e4585b37374
+  data.tar.gz: 6f1b2afeabde54e49e61ac8242b5a97daf3425f64bbe74bd3d0b69259a67d53a3113224273882779f6e12fc4bf97d1851169640c32a8ed56dcae6ce0b14a00f8

data/.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# CODEOWNERS file
+# This file defines who should review code changes in this repository.
+
+* @zendesk/core-gem-owners

data/.github/workflows/ci.yml

@@ -2,21 +2,22 @@ name: CI
 
 on:
   pull_request:
-    branches: ["master"]
+    branches: ["main"]
   push:
-    branches: ["master"]
+    branches: ["main"]
 
 jobs:
   unit-specs:
     runs-on: ubuntu-latest
 
     strategy:
+      fail-fast: false
       matrix:
         ruby-version:
-          - "3.0"
           - "3.1"
           - "3.2"
           - "3.3"
+          - "3.4"
 
     steps:
     - uses: actions/checkout@v4
@@ -30,15 +31,23 @@ jobs:
 
   integration-specs:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - "3.1"
+          - "3.2"
+          - "3.3"
+          - "3.4"
     steps:
     - uses: actions/checkout@v4
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: "3.0"
+        ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true
     - name: Bring up docker-compose stack
-      run: docker-compose up -d
+      run: docker compose up -d
     - name: Build and test with RSpec
       env:
         RACECAR_BROKERS: localhost:9092

data/.github/workflows/publish.yml

@@ -1,12 +1,26 @@
-name: Publish Gem
+name: Publish to RubyGems.org
 
 on:
   push:
-    tags: v*
+    branches: main
+    paths: lib/racecar/version.rb
+  workflow_dispatch:
 
 jobs:
-  call-workflow:
-    uses: zendesk/gw/.github/workflows/ruby-gem-publication.yml@main
-    secrets:
-      RUBY_GEMS_API_KEY: ${{ secrets.RUBY_GEMS_API_KEY }}
-      RUBY_GEMS_TOTP_DEVICE: ${{ secrets.RUBY_GEMS_TOTP_DEVICE }}
+  publish:
+    runs-on: ubuntu-latest
+    environment: rubygems-publish
+    if: github.repository_owner == 'zendesk'
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: false
+          ruby-version: "3.4"
+      - name: Install dependencies
+        run: bundle install
+      - uses: rubygems/release-gem@v1

data/.rspec

@@ -1,3 +1,4 @@
 --format documentation
 --require spec_helper
 --color
+--order random

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+## 2.12.0
+
+* Add tests against Ruby 3.4
+* Drop support for Ruby 3.0
+* Allow rdkafka gem versions newer than 0.15.0
+
 ## 2.11.0
 
 * Allow the liveness probe command to skip loading config files

data/Gemfile

@@ -5,6 +5,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in racecar.gemspec
 gemspec
 
-# We actually support version 7.x (see gemspec); this extra restriction is added just for running the test suite also
-# on Ruby 2.6, which activesupport 7.0 does not support.
-gem 'activesupport', '~> 6.1.0'
+gem 'activesupport', '~> 7.2.0'

data/Gemfile.lock

@@ -1,31 +1,43 @@
 PATH
   remote: .
   specs:
-    racecar (2.11.0)
+    racecar (3.0.0.alpha.1)
       king_konf (~> 1.0.0)
-      rdkafka (~> 0.15.0)
+      rdkafka (>= 0.15.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.7.3)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+    activesupport (7.2.2.1)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
     byebug (11.1.3)
     coderay (1.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.3.4)
+    connection_pool (2.4.1)
     diff-lcs (1.5.0)
     dogstatsd-ruby (5.5.0)
+    drb (2.2.1)
     ffi (1.16.3)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     king_konf (1.0.1)
+    logger (1.6.4)
     method_source (1.0.0)
     mini_portile2 (2.8.5)
-    minitest (5.18.0)
+    minitest (5.25.4)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -33,7 +45,7 @@ GEM
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
     rake (13.0.6)
-    rdkafka (0.15.1)
+    rdkafka (0.21.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
@@ -50,16 +62,16 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
+    securerandom (0.4.1)
     timecop (0.9.6)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.7)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport (~> 6.1.0)
+  activesupport (~> 7.2.0)
   bundler (>= 1.13, < 3)
   dogstatsd-ruby (>= 4.0.0, < 6.0.0)
   pry-byebug
@@ -69,4 +81,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   2.4.9
+   2.6.2

data/README.md

@@ -418,7 +418,7 @@ Racecar supports [Datadog](https://www.datadoghq.com/) monitoring integration. I
 - `datadog_namespace` – The namespace to use for Datadog metrics.
 - `datadog_tags` – Tags that should always be set on Datadog metrics.
 
-Furthermore, there's a [standard Datadog dashboard configuration file](https://raw.githubusercontent.com/zendesk/racecar/master/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
+Furthermore, there's a [standard Datadog dashboard configuration file](https://raw.githubusercontent.com/zendesk/racecar/main/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
 
 #### Consumers Without Rails
 
@@ -716,19 +716,35 @@ apt-get update && apt-get install -y libzstd-dev
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-The integration tests run against a Kafka instance that is not automatically started from within `rspec`. You can set one up using the provided `docker-compose.yml` by running `docker-compose up`.
+The integration tests run against a Kafka instance that is not automatically started from within `rspec`. You can set one up using the provided `docker-compose.yml` by running `docker compose up`.
 
 ### Running RSpec within Docker
 
 There can be behavioural inconsistencies between running the specs on your machine, and in the CI pipeline. Due to this, there is now a Dockerfile included in the project, which is based on the CircleCI ruby 2.7.8 image. This could easily be extended with more Dockerfiles to cover different Ruby versions if desired. In order to run the specs via Docker:
 
 - Uncomment the `tests` service from the docker-compose.yml
-- Bring up the stack with `docker-compose up -d`
-- Execute the entire suite with `docker-compose run --rm tests bundle exec rspec`
-- Execute a single spec or directory with `docker-compose run --rm tests bundle exec rspec spec/integration/consumer_spec.rb`
+- Bring up the stack with `docker compose up -d`
+- Execute the entire suite with `docker compose run --rm tests bundle exec rspec`
+- Execute a single spec or directory with `docker compose run --rm tests bundle exec rspec spec/integration/consumer_spec.rb`
 
 Please note - your code directory is mounted as a volume, so you can make code changes without needing to rebuild
 
+### Releasing a new version
+
+A new version is published to RubyGems.org every time a change to `version.rb` is pushed to the `main` branch.
+In short, follow these steps:
+1. Update `version.rb`,
+2. run `bundle lock` to update `Gemfile.lock`,
+3. merge this change into `main`, and
+4. look at [the action](https://github.com/zendesk/racecar/actions/workflows/publish.yml) for output.
+
+To create a pre-release from a non-main branch:
+1. change the version in `version.rb` to something like `2.13.0.pre.1` or `3.0.0.beta.2`,
+2. push this change to your branch,
+3. go to [Actions → “Publish to RubyGems.org” on GitHub](https://github.com/zendesk/racecar/actions/workflows/publish.yml),
+4. click the “Run workflow” button,
+5. pick your branch from a dropdown.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on [GitHub](https://github.com/zendesk/racecar). Feel free to [join our Slack team](https://ruby-kafka-slack.herokuapp.com/) and ask how best to contribute!

data/Rakefile

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
+require "bundler/setup"
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 
-# Pushing to rubygems is handled by a github workflow
-ENV["gem_push"] = "false"
-
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec

data/docker-compose.yml

@@ -1,8 +1,6 @@
-version: '2.1'
-
 services:
   zookeeper:
-    image: confluentinc/cp-zookeeper:5.5.1
+    image: confluentinc/cp-zookeeper:7.8.1
     ports:
       - "2181:2181"
     environment:
@@ -13,7 +11,7 @@ services:
       test: echo ruok | nc 127.0.0.1 2181 | grep imok
 
   broker:
-    image: confluentinc/cp-kafka:5.5.1
+    image: confluentinc/cp-kafka:7.8.1
     depends_on:
       - zookeeper
     ports:
@@ -57,7 +55,7 @@ services:
   #     RACECAR_BROKERS: broker:29092
   #     DOCKER_SUDO: 'true'
   #   # When bringing up the stack, we just let the container exit. For running the
-  #   # specs, we'll use commands like `docker-compose run tests rspec`
+  #   # specs, we'll use commands like `docker compose run tests rspec`
   #   command: ["echo", "ready"]
   #   volumes:
   #     # The line below allows us to run docker commands from the container itself

data/lib/racecar/async_partition_processor.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require 'racecar/pause'
+require 'concurrent-ruby'
+
+module Racecar
+  class AsyncPartitionProcessor
+    attr_reader :thread
+
+    THREAD_KEY_IDENTIFIER = 'racecar_topic_partition_identifier'.freeze
+
+    def self.thread_key(topic, partition)
+      "#{topic}/#{partition}"
+    end
+
+    def initialize(topic:, partition:, logger:, config:, consumer:, consumer_class:, instrumenter:, rdkafka_consumer:)
+      @topic      = topic
+      @partition  = partition
+      @logger     = logger
+      @config     = config
+      @consumer   = consumer
+      @consumer_class = consumer_class
+      @instrumenter = instrumenter
+      @rdkafka_consumer = rdkafka_consumer
+      @backpressure_paused = Concurrent::AtomicBoolean.new
+      @tpl = build_tpl(topic, partition)
+      setup_async_processing
+    end
+
+    def process(message)
+      push(message)
+    end
+
+    def process_batch(messages)
+      push(messages)
+    end
+
+    def rebalance!
+      processor.rebalance!
+      @queue << nil
+    end
+
+    def shut_down!
+      processor.shut_down!
+      @queue << nil
+    end
+
+    def rebalancing_or_shutting_down?
+      processor.rebalancing_or_shutting_down?
+    end
+
+    def resume_paused_partition
+      processor.resume_paused_partition
+    end
+
+    private
+
+    attr_reader :backpressure_paused, :instrumenter, :consumer_class, :consumer, :queue, :config, :processor, :logger
+
+    def setup_async_processing
+      @processor = PartitionProcessor.new(
+        config: config,
+        logger: logger,
+        instrumenter: instrumenter,
+        consumer_class_instance: consumer_class.new,
+        consumer: consumer,
+        topic: @topic,
+        partition: @partition,
+        pause: Pause.new_from_config(config),
+        rdkafka_consumer: @rdkafka_consumer,
+      )
+      @queue  = Queue.new
+      @thread = nil
+
+      use_process_batch = consumer_class.method_defined?(:process_batch)
+
+      if use_process_batch
+        spawn_thread do |msgs|
+          processor.process_batch(msgs)
+        end
+      else
+        spawn_thread do |msgs|
+          msgs.each do |msg|
+            processor.process(msg)
+          end
+        end
+      end
+    end
+
+    def spawn_thread(&block)
+      @thread = Thread.new do
+        Thread.current.name = "Racecar thread for #{thread_key}"
+        Thread.current[AsyncPartitionProcessor::THREAD_KEY_IDENTIFIER] = thread_key
+        main_processing_loop(block)
+      end
+    end
+
+    def push(messages)
+      @queue << Array(messages)
+      maybe_apply_backpressure
+    end
+
+    def maybe_apply_backpressure
+      if @backpressure_paused.false? && @queue.size >= config.multithreaded_processing_max_queue_size
+        @backpressure_paused.make_true
+        @rdkafka_consumer.pause(@tpl)
+        logger.debug "Paused partition #{@topic}/#{@partition}: queue reached capacity (#{@queue.size}/#{config.multithreaded_processing_max_queue_size})"
+      end
+    end
+
+    def maybe_resume_the_partition
+      if @backpressure_paused.true? && @queue.size < config.multithreaded_processing_resume_threshold * config.multithreaded_processing_max_queue_size
+        @backpressure_paused.make_false
+        @rdkafka_consumer.resume(@tpl)
+      end
+    end
+
+    def build_tpl(topic, partition)
+      Rdkafka::Consumer::TopicPartitionList.new.tap do |tpl|
+        tpl.add_topic_and_partitions_with_offsets(topic, partition => -1001)
+      end
+    end
+
+    def thread_key
+      self.class.thread_key(@topic, @partition)
+    end
+
+    def main_processing_loop(block)
+      loop do
+        msgs = @queue.pop
+        break if msgs.nil?
+
+        maybe_resume_the_partition
+        block.call(msgs)
+      rescue => e
+        logger.error "Error in processing thread for #{thread_key}: #{e.class} - #{e.full_message}. backtrace: #{e.backtrace&.first(10)&.join("\n")}"
+      end
+    ensure
+      @processor.teardown
+    end
+  end
+end

data/lib/racecar/cli.rb

@@ -65,8 +65,7 @@ module Racecar
         config.install_liveness_probe
       end
 
-      processor = consumer_class.new
-      @runner = Racecar.runner(processor)
+      @runner = Racecar.runner(consumer_class)
       @runner.run
       nil
     end

data/lib/racecar/config.rb

@@ -194,6 +194,18 @@ module Racecar
     desc "Strategy for switching topics when there are multiple subscriptions. `exhaust-topic` will only switch when the consumer poll returns no messages. `round-robin` will switch after each poll regardless.\nWarning: `round-robin` will be the default in Racecar 3.x"
     string :multi_subscription_strategy, allowed_values: %w(round-robin exhaust-topic), default: "exhaust-topic"
 
+    desc "Whether multithreaded processing is enabled"
+    boolean :multithreaded_processing_enabled, default: false
+
+    desc "Max size of the queue of messages waiting to be processed when multithreaded processing is enabled"
+    integer :multithreaded_processing_max_queue_size, default: 1000
+
+    desc "Timeout in seconds for the main thread to wait for a processing thread to finish when shutting down the consumer with multithreaded processing enabled"
+    integer :multithreaded_processing_shutdown_timeout, default: 300
+
+    desc "Multi threaded queue resume threshold as a percentage of `multithreaded_processing_max_queue_size`. Defaults to 0.5, meaning that the consumer will attempt to resume a paused partition when the queue size drops below 50% of the max queue size."
+    float :multithreaded_processing_resume_threshold, default: 0.5
+
     # The error handler must be set directly on the object.
     attr_reader :error_handler
 

data/lib/racecar/consumer.rb

@@ -68,16 +68,16 @@ module Racecar
         @instrumenter.instrument('deliver_messages', instrumentation_payload) do
           @delivery_handles.each do |handle|
             begin
-              # rdkafka-ruby checks every wait_timeout seconds if the message was
-              # successfully delivered, up to max_wait_timeout seconds before raising
-              # Rdkafka::AbstractHandle::WaitTimeoutError. librdkafka will (re)try to
-              # deliver all messages in the background, until "config.message_timeout"
+              # rdkafka-ruby checks with exponential backoff starting at 0 seconds wait
+              # if the message was successfully delivered, up to max_wait_timeout seconds
+              # before raising Rdkafka::AbstractHandle::WaitTimeoutError. librdkafka will 
+              # (re)try to deliver all messages in the background, until "config.message_timeout"
               # (message.timeout.ms) is exceeded. Phrased differently, rdkafka-ruby's
               # WaitTimeoutError is just informative.
               # The raising can be avoided if max_wait_timeout below is greater than
               # config.message_timeout, but config is not available here (without
               # changing the interface).
-              handle.wait(max_wait_timeout: 60, wait_timeout: 0.1)
+              handle.wait(max_wait_timeout: 60)
             rescue Rdkafka::AbstractHandle::WaitTimeoutError => e
               partition = MessageDeliveryError.partition_from_delivery_handle(handle)
               # ideally we could use the logger passed to the Runner, but it is not

data/lib/racecar/consumer_set.rb

@@ -1,21 +1,24 @@
 # frozen_string_literal: true
 
+require "racecar/delivery_callback"
+
 module Racecar
   class ConsumerSet
     MAX_POLL_TRIES = 10
 
-    def initialize(config, logger, instrumenter = NullInstrumenter)
+    def initialize(config, logger, partition_processors, instrumenter = NullInstrumenter)
       @config, @logger = config, logger
       @instrumenter = instrumenter
+      @partition_processors = partition_processors
       raise ArgumentError, "Subscriptions must not be empty when subscribing" if @config.subscriptions.empty?
 
       @consumers = []
       @consumer_id_iterator = (0...@config.subscriptions.size).cycle
+      @producer_mutex = Mutex.new
 
       @previous_retries = 0
 
       @last_poll_read_nil_message = false
-      @paused_tpls = Hash.new { |h, k| h[k] = {} }
     end
 
     def poll(max_wait_time_ms = @config.max_wait_time_ms)
@@ -48,8 +51,9 @@ module Racecar
       messages
     end
 
-    def store_offset(message)
-      current.store_offset(message)
+    def store_offset(message, raw_consumer = nil)
+      consumer = raw_consumer || current
+      consumer.store_offset(message)
     rescue Rdkafka::RdkafkaError => e
       if e.code == :state # -172
         @logger.warn "Attempted to store_offset, but we're not subscribed to it: #{ErroneousStateError.new(e)}"
@@ -66,13 +70,28 @@ module Racecar
 
     def close
       each_subscribed(&:close)
-      @paused_tpls.clear
+      reset_producer!
+    end
+
+    def producer
+      @producer_mutex.synchronize do
+        @producer ||= Rdkafka::Config.new(producer_config).producer.tap do |p|
+          p.delivery_callback = Racecar::DeliveryCallback.new(instrumenter: @instrumenter)
+        end
+      end
+    end
+
+    def reset_producer!
+      @producer_mutex.synchronize do
+        @producer&.close
+        @producer = nil
+      end
     end
 
     def current
       @consumers[@consumer_id_iterator.peek] ||= begin
         consumer_config = Rdkafka::Config.new(rdkafka_config(current_subscription))
-        listener = RebalanceListener.new(@config.consumer_class, @instrumenter)
+        listener = RebalanceListener.new(@config, @instrumenter, @partition_processors)
         consumer_config.consumer_rebalance_listener = listener
         consumer = consumer_config.consumer
         listener.rdkafka_consumer = consumer
@@ -86,44 +105,38 @@ module Racecar
 
     def each_subscribed
       if block_given?
-        @consumers.each { |c| yield c }
+        @consumers.compact.each { |c| yield c }
       else
-        @consumers.each
+        @consumers.compact.each
       end
     end
 
-    def pause(topic, partition, offset)
+    def pause(topic, partition, offset = nil)
       consumer, filtered_tpl = find_consumer_by(topic, partition)
-      if !consumer
+      unless consumer
         @logger.info "Attempted to pause #{topic}/#{partition}, but we're not subscribed to it"
         return
       end
 
       consumer.pause(filtered_tpl)
-      fake_msg = OpenStruct.new(topic: topic, partition: partition, offset: offset)
-      consumer.seek(fake_msg)
-
-      @paused_tpls[topic][partition] = [consumer, filtered_tpl]
+      if offset
+        fake_msg = OpenStruct.new(topic: topic, partition: partition, offset: offset)
+        consumer.seek(fake_msg)
+      end
     end
 
     def resume(topic, partition)
       consumer, filtered_tpl = find_consumer_by(topic, partition)
 
-      if !consumer && @paused_tpls[topic][partition]
-        consumer, filtered_tpl = @paused_tpls[topic][partition]
-      end
-
-      if !consumer
+      unless consumer
         @logger.info "Attempted to resume #{topic}/#{partition}, but we're not subscribed to it"
         return
       end
 
       consumer.resume(filtered_tpl)
-      @paused_tpls[topic].delete(partition)
-      @paused_tpls.delete(topic) if @paused_tpls[topic].empty?
     end
 
-    alias :each :each_subscribed
+  alias :each :each_subscribed
 
     # Subscribe to all topics eagerly, even if there's still messages elsewhere. Usually
     # that's not needed and Kafka might rebalance if topics are not polled frequently
@@ -269,5 +282,18 @@ module Racecar
       r = limit_ms - ((Time.now - started_at_time)*1000).round
       r <= 0 ? 0 : r
     end
+
+    def producer_config
+      cfg = {
+        "bootstrap.servers"      => @config.brokers.join(","),
+        "client.id"              => @config.client_id,
+        "statistics.interval.ms" => @config.statistics_interval_ms,
+        "message.timeout.ms"     => @config.message_timeout * 1000,
+        "partitioner"            => @config.partitioner.to_s,
+      }
+      cfg["compression.codec"] = @config.producer_compression_codec.to_s unless @config.producer_compression_codec.nil?
+      cfg.merge!(@config.rdkafka_producer)
+      cfg
+    end
   end
 end