racecar 2.12.0 → 3.0.0.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4e6b0c4d52637bd8c6bb2c5e3d69013889053973195a5d73ac5d5645047f985
-  data.tar.gz: 23425d552932665cafa03cf245d3d4ef879e053e9e9c60d1eb172455f405f3f3
+  metadata.gz: 8d48006d6ad7a9f1cb57902edbd8a2785f3e05c629a71c3345238a0841abeda6
+  data.tar.gz: 06b557c5757e1c4bda2d1d6cebd2e679ab27053bc02f58636911f8df8d67f217
 SHA512:
-  metadata.gz: 62d52be41cb3821a5d6534fea43deea8d02dd8a8f03d3449249be0fcdcbf5c842cbb2be8aac7065b97fb515b16370ab93f114ab44dd85dce6a22091d9ac04de4
-  data.tar.gz: 006efc7649ddb4257a80e1f2400a62e9e9cba0fa5573d3947f47293d4d77b9b701d66ebf6d465cf91fbf209771e5699ddc9cd7df238ecddf7f54ffbab4eb0c15
+  metadata.gz: 9e6cd591d16f776a84140105fd70eb565c70c6001ac03c5f61ea0ef45b40a8fd29e656a8da4b445894c87d9bb6002376364d8de896af481847222e4585b37374
+  data.tar.gz: 6f1b2afeabde54e49e61ac8242b5a97daf3425f64bbe74bd3d0b69259a67d53a3113224273882779f6e12fc4bf97d1851169640c32a8ed56dcae6ce0b14a00f8

data/.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# CODEOWNERS file
+# This file defines who should review code changes in this repository.
+
+* @zendesk/core-gem-owners

data/.github/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   pull_request:
-    branches: ["master"]
+    branches: ["main"]
   push:
-    branches: ["master"]
+    branches: ["main"]
 
 jobs:
   unit-specs:

data/.github/workflows/publish.yml

@@ -1,12 +1,26 @@
-name: Publish Gem
+name: Publish to RubyGems.org
 
 on:
   push:
-    tags: v*
+    branches: main
+    paths: lib/racecar/version.rb
+  workflow_dispatch:
 
 jobs:
-  call-workflow:
-    uses: zendesk/gw/.github/workflows/ruby-gem-publication.yml@main
-    secrets:
-      RUBY_GEMS_API_KEY: ${{ secrets.RUBY_GEMS_API_KEY }}
-      RUBY_GEMS_TOTP_DEVICE: ${{ secrets.RUBY_GEMS_TOTP_DEVICE }}
+  publish:
+    runs-on: ubuntu-latest
+    environment: rubygems-publish
+    if: github.repository_owner == 'zendesk'
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: false
+          ruby-version: "3.4"
+      - name: Install dependencies
+        run: bundle install
+      - uses: rubygems/release-gem@v1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    racecar (2.12.0)
+    racecar (3.0.0.alpha.1)
       king_konf (~> 1.0.0)
       rdkafka (>= 0.15.0)
 
@@ -45,7 +45,7 @@ GEM
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
     rake (13.0.6)
-    rdkafka (0.18.0)
+    rdkafka (0.21.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)

data/README.md

@@ -418,7 +418,7 @@ Racecar supports [Datadog](https://www.datadoghq.com/) monitoring integration. I
 - `datadog_namespace` – The namespace to use for Datadog metrics.
 - `datadog_tags` – Tags that should always be set on Datadog metrics.
 
-Furthermore, there's a [standard Datadog dashboard configuration file](https://raw.githubusercontent.com/zendesk/racecar/master/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
+Furthermore, there's a [standard Datadog dashboard configuration file](https://raw.githubusercontent.com/zendesk/racecar/main/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
 
 #### Consumers Without Rails
 
@@ -729,6 +729,22 @@ There can be behavioural inconsistencies between running the specs on your machi
 
 Please note - your code directory is mounted as a volume, so you can make code changes without needing to rebuild
 
+### Releasing a new version
+
+A new version is published to RubyGems.org every time a change to `version.rb` is pushed to the `main` branch.
+In short, follow these steps:
+1. Update `version.rb`,
+2. run `bundle lock` to update `Gemfile.lock`,
+3. merge this change into `main`, and
+4. look at [the action](https://github.com/zendesk/racecar/actions/workflows/publish.yml) for output.
+
+To create a pre-release from a non-main branch:
+1. change the version in `version.rb` to something like `2.13.0.pre.1` or `3.0.0.beta.2`,
+2. push this change to your branch,
+3. go to [Actions → “Publish to RubyGems.org” on GitHub](https://github.com/zendesk/racecar/actions/workflows/publish.yml),
+4. click the “Run workflow” button,
+5. pick your branch from a dropdown.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on [GitHub](https://github.com/zendesk/racecar). Feel free to [join our Slack team](https://ruby-kafka-slack.herokuapp.com/) and ask how best to contribute!

data/Rakefile

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
+require "bundler/setup"
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 
-# Pushing to rubygems is handled by a github workflow
-ENV["gem_push"] = "false"
-
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec

data/lib/racecar/async_partition_processor.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require 'racecar/pause'
+require 'concurrent-ruby'
+
+module Racecar
+  class AsyncPartitionProcessor
+    attr_reader :thread
+
+    THREAD_KEY_IDENTIFIER = 'racecar_topic_partition_identifier'.freeze
+
+    def self.thread_key(topic, partition)
+      "#{topic}/#{partition}"
+    end
+
+    def initialize(topic:, partition:, logger:, config:, consumer:, consumer_class:, instrumenter:, rdkafka_consumer:)
+      @topic      = topic
+      @partition  = partition
+      @logger     = logger
+      @config     = config
+      @consumer   = consumer
+      @consumer_class = consumer_class
+      @instrumenter = instrumenter
+      @rdkafka_consumer = rdkafka_consumer
+      @backpressure_paused = Concurrent::AtomicBoolean.new
+      @tpl = build_tpl(topic, partition)
+      setup_async_processing
+    end
+
+    def process(message)
+      push(message)
+    end
+
+    def process_batch(messages)
+      push(messages)
+    end
+
+    def rebalance!
+      processor.rebalance!
+      @queue << nil
+    end
+
+    def shut_down!
+      processor.shut_down!
+      @queue << nil
+    end
+
+    def rebalancing_or_shutting_down?
+      processor.rebalancing_or_shutting_down?
+    end
+
+    def resume_paused_partition
+      processor.resume_paused_partition
+    end
+
+    private
+
+    attr_reader :backpressure_paused, :instrumenter, :consumer_class, :consumer, :queue, :config, :processor, :logger
+
+    def setup_async_processing
+      @processor = PartitionProcessor.new(
+        config: config,
+        logger: logger,
+        instrumenter: instrumenter,
+        consumer_class_instance: consumer_class.new,
+        consumer: consumer,
+        topic: @topic,
+        partition: @partition,
+        pause: Pause.new_from_config(config),
+        rdkafka_consumer: @rdkafka_consumer,
+      )
+      @queue  = Queue.new
+      @thread = nil
+
+      use_process_batch = consumer_class.method_defined?(:process_batch)
+
+      if use_process_batch
+        spawn_thread do |msgs|
+          processor.process_batch(msgs)
+        end
+      else
+        spawn_thread do |msgs|
+          msgs.each do |msg|
+            processor.process(msg)
+          end
+        end
+      end
+    end
+
+    def spawn_thread(&block)
+      @thread = Thread.new do
+        Thread.current.name = "Racecar thread for #{thread_key}"
+        Thread.current[AsyncPartitionProcessor::THREAD_KEY_IDENTIFIER] = thread_key
+        main_processing_loop(block)
+      end
+    end
+
+    def push(messages)
+      @queue << Array(messages)
+      maybe_apply_backpressure
+    end
+
+    def maybe_apply_backpressure
+      if @backpressure_paused.false? && @queue.size >= config.multithreaded_processing_max_queue_size
+        @backpressure_paused.make_true
+        @rdkafka_consumer.pause(@tpl)
+        logger.debug "Paused partition #{@topic}/#{@partition}: queue reached capacity (#{@queue.size}/#{config.multithreaded_processing_max_queue_size})"
+      end
+    end
+
+    def maybe_resume_the_partition
+      if @backpressure_paused.true? && @queue.size < config.multithreaded_processing_resume_threshold * config.multithreaded_processing_max_queue_size
+        @backpressure_paused.make_false
+        @rdkafka_consumer.resume(@tpl)
+      end
+    end
+
+    def build_tpl(topic, partition)
+      Rdkafka::Consumer::TopicPartitionList.new.tap do |tpl|
+        tpl.add_topic_and_partitions_with_offsets(topic, partition => -1001)
+      end
+    end
+
+    def thread_key
+      self.class.thread_key(@topic, @partition)
+    end
+
+    def main_processing_loop(block)
+      loop do
+        msgs = @queue.pop
+        break if msgs.nil?
+
+        maybe_resume_the_partition
+        block.call(msgs)
+      rescue => e
+        logger.error "Error in processing thread for #{thread_key}: #{e.class} - #{e.full_message}. backtrace: #{e.backtrace&.first(10)&.join("\n")}"
+      end
+    ensure
+      @processor.teardown
+    end
+  end
+end

data/lib/racecar/cli.rb

@@ -65,8 +65,7 @@ module Racecar
         config.install_liveness_probe
       end
 
-      processor = consumer_class.new
-      @runner = Racecar.runner(processor)
+      @runner = Racecar.runner(consumer_class)
       @runner.run
       nil
     end

data/lib/racecar/config.rb

@@ -194,6 +194,18 @@ module Racecar
     desc "Strategy for switching topics when there are multiple subscriptions. `exhaust-topic` will only switch when the consumer poll returns no messages. `round-robin` will switch after each poll regardless.\nWarning: `round-robin` will be the default in Racecar 3.x"
     string :multi_subscription_strategy, allowed_values: %w(round-robin exhaust-topic), default: "exhaust-topic"
 
+    desc "Whether multithreaded processing is enabled"
+    boolean :multithreaded_processing_enabled, default: false
+
+    desc "Max size of the queue of messages waiting to be processed when multithreaded processing is enabled"
+    integer :multithreaded_processing_max_queue_size, default: 1000
+
+    desc "Timeout in seconds for the main thread to wait for a processing thread to finish when shutting down the consumer with multithreaded processing enabled"
+    integer :multithreaded_processing_shutdown_timeout, default: 300
+
+    desc "Multi threaded queue resume threshold as a percentage of `multithreaded_processing_max_queue_size`. Defaults to 0.5, meaning that the consumer will attempt to resume a paused partition when the queue size drops below 50% of the max queue size."
+    float :multithreaded_processing_resume_threshold, default: 0.5
+
     # The error handler must be set directly on the object.
     attr_reader :error_handler
 

data/lib/racecar/consumer_set.rb

@@ -1,21 +1,24 @@
 # frozen_string_literal: true
 
+require "racecar/delivery_callback"
+
 module Racecar
   class ConsumerSet
     MAX_POLL_TRIES = 10
 
-    def initialize(config, logger, instrumenter = NullInstrumenter)
+    def initialize(config, logger, partition_processors, instrumenter = NullInstrumenter)
       @config, @logger = config, logger
       @instrumenter = instrumenter
+      @partition_processors = partition_processors
       raise ArgumentError, "Subscriptions must not be empty when subscribing" if @config.subscriptions.empty?
 
       @consumers = []
       @consumer_id_iterator = (0...@config.subscriptions.size).cycle
+      @producer_mutex = Mutex.new
 
       @previous_retries = 0
 
       @last_poll_read_nil_message = false
-      @paused_tpls = Hash.new { |h, k| h[k] = {} }
     end
 
     def poll(max_wait_time_ms = @config.max_wait_time_ms)
@@ -48,8 +51,9 @@ module Racecar
       messages
     end
 
-    def store_offset(message)
-      current.store_offset(message)
+    def store_offset(message, raw_consumer = nil)
+      consumer = raw_consumer || current
+      consumer.store_offset(message)
     rescue Rdkafka::RdkafkaError => e
       if e.code == :state # -172
         @logger.warn "Attempted to store_offset, but we're not subscribed to it: #{ErroneousStateError.new(e)}"
@@ -66,13 +70,28 @@ module Racecar
 
     def close
       each_subscribed(&:close)
-      @paused_tpls.clear
+      reset_producer!
+    end
+
+    def producer
+      @producer_mutex.synchronize do
+        @producer ||= Rdkafka::Config.new(producer_config).producer.tap do |p|
+          p.delivery_callback = Racecar::DeliveryCallback.new(instrumenter: @instrumenter)
+        end
+      end
+    end
+
+    def reset_producer!
+      @producer_mutex.synchronize do
+        @producer&.close
+        @producer = nil
+      end
     end
 
     def current
       @consumers[@consumer_id_iterator.peek] ||= begin
         consumer_config = Rdkafka::Config.new(rdkafka_config(current_subscription))
-        listener = RebalanceListener.new(@config.consumer_class, @instrumenter)
+        listener = RebalanceListener.new(@config, @instrumenter, @partition_processors)
         consumer_config.consumer_rebalance_listener = listener
         consumer = consumer_config.consumer
         listener.rdkafka_consumer = consumer
@@ -86,44 +105,38 @@ module Racecar
 
     def each_subscribed
       if block_given?
-        @consumers.each { |c| yield c }
+        @consumers.compact.each { |c| yield c }
       else
-        @consumers.each
+        @consumers.compact.each
       end
     end
 
-    def pause(topic, partition, offset)
+    def pause(topic, partition, offset = nil)
       consumer, filtered_tpl = find_consumer_by(topic, partition)
-      if !consumer
+      unless consumer
         @logger.info "Attempted to pause #{topic}/#{partition}, but we're not subscribed to it"
         return
       end
 
       consumer.pause(filtered_tpl)
-      fake_msg = OpenStruct.new(topic: topic, partition: partition, offset: offset)
-      consumer.seek(fake_msg)
-
-      @paused_tpls[topic][partition] = [consumer, filtered_tpl]
+      if offset
+        fake_msg = OpenStruct.new(topic: topic, partition: partition, offset: offset)
+        consumer.seek(fake_msg)
+      end
     end
 
     def resume(topic, partition)
       consumer, filtered_tpl = find_consumer_by(topic, partition)
 
-      if !consumer && @paused_tpls[topic][partition]
-        consumer, filtered_tpl = @paused_tpls[topic][partition]
-      end
-
-      if !consumer
+      unless consumer
         @logger.info "Attempted to resume #{topic}/#{partition}, but we're not subscribed to it"
         return
       end
 
       consumer.resume(filtered_tpl)
-      @paused_tpls[topic].delete(partition)
-      @paused_tpls.delete(topic) if @paused_tpls[topic].empty?
     end
 
-    alias :each :each_subscribed
+  alias :each :each_subscribed
 
     # Subscribe to all topics eagerly, even if there's still messages elsewhere. Usually
     # that's not needed and Kafka might rebalance if topics are not polled frequently
@@ -269,5 +282,18 @@ module Racecar
       r = limit_ms - ((Time.now - started_at_time)*1000).round
       r <= 0 ? 0 : r
     end
+
+    def producer_config
+      cfg = {
+        "bootstrap.servers"      => @config.brokers.join(","),
+        "client.id"              => @config.client_id,
+        "statistics.interval.ms" => @config.statistics_interval_ms,
+        "message.timeout.ms"     => @config.message_timeout * 1000,
+        "partitioner"            => @config.partitioner.to_s,
+      }
+      cfg["compression.codec"] = @config.producer_compression_codec.to_s unless @config.producer_compression_codec.nil?
+      cfg.merge!(@config.rdkafka_producer)
+      cfg
+    end
   end
 end

data/lib/racecar/partition_processor.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "rdkafka"
+require "racecar/pause"
+require "racecar/delivery_callback"
+
+module Racecar
+  class PartitionProcessor
+    attr_reader :consumer_class_instance, :config, :logger, :instrumenter, :consumer, :topic, :partition, :pause
+    attr_accessor :rebalancing, :shutting_down
+
+    def initialize(config:, logger:, instrumenter:, consumer_class_instance:, consumer:, topic:, partition:, pause:, rdkafka_consumer: nil)
+      @config = config
+      @logger = logger
+      @instrumenter = instrumenter
+      @consumer_class_instance = consumer_class_instance
+      @pause = pause
+      @topic = topic
+      @partition = partition
+      @consumer = consumer
+      @rdkafka_consumer = rdkafka_consumer
+
+      if config.multithreaded_processing_enabled
+        consumer_class_instance.configure(
+          producer:     consumer.producer,
+          consumer:     @consumer,
+          instrumenter: @instrumenter,
+          config:       @config,
+        )
+      end
+
+      @sleep_mutex = Mutex.new
+      @sleep_cv = ConditionVariable.new
+    end
+
+    def process(message)
+      payload = {
+        consumer_class: consumer_class_instance.class.to_s,
+        topic:          message.topic,
+        partition:      message.partition,
+        offset:         message.offset,
+        create_time:    message.timestamp,
+        key:            message.key,
+        value:          message.payload,
+        headers:        message.headers,
+      }
+      @instrumenter.instrument("start_process_message", payload)
+
+      with_error_handling(message, payload) do |pause|
+        @instrumenter.instrument("process_message", payload) do
+          if @config.multithreaded_processing_enabled && consumer_class_instance.instance_variable_get(:@producer)&.closed?
+            reconfigure_consumer_class_instance!
+          end
+          consumer_class_instance.process(Racecar::Message.new(message, retries_count: pause.pauses_count))
+          consumer_class_instance.deliver!
+          consumer.store_offset(message, @rdkafka_consumer) unless rebalancing
+        end
+      end
+    end
+
+    def process_batch(messages)
+      first, last = messages.first, messages.last
+      payload = {
+        consumer_class:   consumer_class_instance.class.to_s,
+        topic:            first.topic,
+        partition:        first.partition,
+        first_offset:     first.offset,
+        last_offset:      last.offset,
+        last_create_time: last.timestamp,
+        message_count:    messages.size,
+      }
+      @instrumenter.instrument("start_process_batch", payload)
+
+      with_error_handling(messages, payload) do |pause|
+        @instrumenter.instrument("process_batch", payload) do
+          racecar_messages = messages.map do |message|
+            Racecar::Message.new(message, retries_count: pause.pauses_count)
+          end
+          if @config.multithreaded_processing_enabled && consumer_class_instance.instance_variable_get(:@producer)&.closed?
+            reconfigure_consumer_class_instance!
+          end
+          consumer_class_instance.process_batch(racecar_messages)
+          consumer_class_instance.deliver!
+          consumer.store_offset(messages.last, @rdkafka_consumer) unless rebalancing
+        end
+      end
+    end
+
+    def teardown
+      consumer_class_instance.deliver! unless rebalancing
+    ensure
+      consumer_class_instance.teardown
+    end
+
+    def resume_paused_partition
+      return if config.pause_timeout == 0 || !pause.paused?
+
+      @instrumenter.instrument("pause_status", {
+        topic:          topic,
+        partition:      partition,
+        duration:       pause.pause_duration,
+        consumer_class: consumer_class_instance.class.to_s,
+      })
+
+      if pause.paused? && pause.expired?
+        logger.info "Automatically resuming partition #{topic}/#{partition}, pause timeout expired"
+        consumer.resume(topic, partition)
+        pause.resume!
+      end
+    end
+
+    def rebalance!
+      @rebalancing = true
+      @sleep_mutex.synchronize { @sleep_cv.signal }
+    end
+
+    def shut_down!
+      @shutting_down = true
+      @sleep_mutex.synchronize { @sleep_cv.signal }
+      resume_paused_partition
+    end
+
+    def rebalancing_or_shutting_down?
+      rebalancing || shutting_down
+    end
+
+    private
+
+    def with_error_handling(messages, payload)
+      if config.multithreaded_processing_enabled
+        with_multi_threaded_error_handling(messages, payload) { |pause| yield(pause) }
+      else
+        with_single_threaded_error_handling(messages, payload) { |pause| yield(pause) }
+      end
+    end
+
+    def with_multi_threaded_error_handling(messages, payload)
+      loop do
+        begin
+          yield(pause)
+          pause.reset!
+          break
+        rescue => e
+          if rebalancing
+            Thread.exit
+          elsif !shutting_down
+            handle_processing_error(e, payload, pause: pause)
+            pause.pause!
+            unless config.pause_timeout <= 0
+              @sleep_mutex.synchronize do
+                next if rebalancing || shutting_down
+                @sleep_cv.wait(@sleep_mutex, pause.backoff_interval)
+              end
+            end
+            Thread.exit if rebalancing
+            break if shutting_down || config.pause_timeout <= 0
+          else
+            handle_processing_error(e, payload, pause: pause)
+            break
+          end
+        end
+      end
+    end
+
+    def with_single_threaded_error_handling(messages, payload)
+      offsets = messages.is_a?(Array) ? messages.first.offset..messages.last.offset : messages.offset..messages.offset
+      with_pause(offsets) do
+        yield(pause)
+      rescue => e
+        handle_processing_error(e, payload, pause: pause)
+        raise e
+      end
+    end
+
+    def with_pause(offsets)
+      return yield if config.pause_timeout == 0
+
+      begin
+        yield
+        pause.reset!
+      rescue => e
+        desc = "#{topic}/#{partition}"
+        logger.error "Failed to process #{desc} at #{offsets}: #{e}"
+        logger.warn "Pausing partition #{desc} for #{pause.backoff_interval} seconds"
+        consumer.pause(topic, partition, offsets.first)
+        pause.pause!
+      end
+    end
+
+    def handle_processing_error(error, payload, pause:)
+      if error.is_a?(Racecar::MessageDeliveryError) && error.code == :msg_timed_out
+        logger.error error.to_s
+        logger.error "Racecar will reset the producer to force a new broker connection."
+        reset_producer!
+        payload[:unrecoverable_delivery_error] = true
+      else
+        payload[:unrecoverable_delivery_error] = false
+      end
+      payload[:retries_count] = pause.pauses_count
+      config.error_handler.call(error, payload)
+    end
+
+    def reset_producer!
+      consumer.reset_producer!
+      reconfigure_consumer_class_instance!
+    end
+
+    def reconfigure_consumer_class_instance!
+      consumer_class_instance.configure(
+        producer:     consumer.producer,
+        consumer:     consumer,
+        instrumenter: @instrumenter,
+        config:       @config,
+      )
+    end
+  end
+end

data/lib/racecar/pause.rb

@@ -4,6 +4,22 @@ module Racecar
   class Pause
     attr_reader :pauses_count
 
+    def self.new_from_config(config)
+      timeout = if config.pause_timeout == -1 || config.pause_timeout == 0
+                  nil
+                elsif config.pause_timeout > 0
+                  config.pause_timeout
+                else
+                  raise ArgumentError, "Invalid value for pause_timeout: must be integer greater or equal -1"
+                end
+
+      new(
+        timeout:             timeout,
+        max_timeout:         config.max_pause_timeout,
+        exponential_backoff: config.pause_with_exponential_backoff
+      )
+    end
+
     def initialize(timeout: nil, max_timeout: nil, exponential_backoff: false)
       @started_at = nil
       @pauses_count = 0

data/lib/racecar/rebalance_listener.rb

@@ -1,8 +1,10 @@
 module Racecar
   class RebalanceListener
-    def initialize(consumer_class, instrumenter)
-      @consumer_class = consumer_class
+    def initialize(config, instrumenter, partition_processors)
+      @consumer_class = config.consumer_class
+      @config = config
       @instrumenter = instrumenter
+      @partition_processors = partition_processors
       @rdkafka_consumer = nil
     end
 
@@ -24,6 +26,14 @@ module Racecar
 
       instrument("partitions_revoked", partitions: event.partition_numbers) do
         consumer_class.on_partitions_revoked(event)
+        rdkafka_topic_partition_list.to_h.each do |topic, partitions_metadata|
+          partitions_metadata.flatten.map(&:partition).each do |partition|
+            key = Runner.topic_partition_key(topic, partition)
+            processor = @partition_processors[key]
+            processor&.rebalance!
+            @partition_processors.delete(key)
+          end
+        end
       end
     end
 

data/lib/racecar/runner.rb

@@ -6,156 +6,120 @@ require "racecar/message"
 require "racecar/message_delivery_error"
 require "racecar/erroneous_state_error"
 require "racecar/delivery_callback"
+require "racecar/partition_processor"
+require "racecar/async_partition_processor"
 
 module Racecar
   class Runner
-    attr_reader :processor, :config, :logger
+    attr_reader :consumer_class, :config, :logger, :partition_processors
 
-    def initialize(processor, config:, logger:, instrumenter: NullInstrumenter)
-      @processor, @config, @logger = processor, config, logger
+    def initialize(consumer_class, config:, logger:, instrumenter: NullInstrumenter)
+      @consumer_class, @config, @logger = consumer_class, config, logger
       @instrumenter = instrumenter
       @stop_requested = false
-      Rdkafka::Config.logger = logger
-
-      if processor.respond_to?(:statistics_callback)
-        Rdkafka::Config.statistics_callback = processor.method(:statistics_callback).to_proc
+      @partition_processors = Concurrent::Hash.new
+      @consumer_class_instance = consumer_class.new
+      if @consumer_class_instance.respond_to?(:statistics_callback) && Rdkafka::Config.statistics_callback.nil?
+        Rdkafka::Config.statistics_callback = @consumer_class_instance.method(:statistics_callback).to_proc
       end
-
-      setup_pauses
+      Rdkafka::Config.logger = logger
     end
 
-    def setup_pauses
-      timeout = if config.pause_timeout == -1
-        nil
-      elsif config.pause_timeout == 0
-        # no op, handled elsewhere
-      elsif config.pause_timeout > 0
-        config.pause_timeout
-      else
-        raise ArgumentError, "Invalid value for pause_timeout: must be integer greater or equal -1"
-      end
-
-      @pauses = Hash.new {|h, k|
-        h[k] = Hash.new {|h2, k2|
-          h2[k2] = ::Racecar::Pause.new(
-            timeout:             timeout,
-            max_timeout:         config.max_pause_timeout,
-            exponential_backoff: config.pause_with_exponential_backoff
-          )
-        }
-      }
+    def self.topic_partition_key(topic, partition)
+      "#{topic}/#{partition}"
     end
 
     def run
       install_signal_handlers
       @stop_requested = false
 
-      # Configure the consumer with a producer so it can produce messages and
-      # with a consumer so that it can support advanced use-cases.
-      processor.configure(
-        producer:     producer,
-        consumer:     consumer,
-        instrumenter: @instrumenter,
-        config:       @config,
-      )
+      unless config.multithreaded_processing_enabled
+        @consumer_class_instance.configure(
+          producer:     consumer.producer,
+          consumer:     consumer,
+          instrumenter: @instrumenter,
+          config:       config,
+        )
+      end
 
-      instrumentation_payload = {
-        consumer_class: processor.class.to_s,
+      loop_payload = {
+        consumer_class: consumer_class.to_s,
         consumer_set: consumer
       }
-
       # Main loop
-      loop do
-        break if @stop_requested
-        resume_paused_partitions
-
-        @instrumenter.instrument("start_main_loop", instrumentation_payload)
-        @instrumenter.instrument("main_loop", instrumentation_payload) do
-          case process_method
-          when :batch then
-            msg_per_part = consumer.batch_poll(config.max_wait_time_ms).group_by(&:partition)
-            msg_per_part.each_value do |messages|
-              process_batch(messages)
+      begin
+        loop do
+          break if @stop_requested
+
+          @instrumenter.instrument("start_main_loop", loop_payload)
+          @instrumenter.instrument("main_loop", loop_payload) do
+            resume_all_paused_partitions unless config.multithreaded_processing_enabled
+
+            case process_method
+            when :batch then
+              msg_per_part = consumer.batch_poll(config.max_wait_time_ms).group_by(&:partition)
+              msg_per_part.each_value do |messages_per_partition|
+                processor = assign_and_get_processor(messages_per_partition)
+                processor&.process_batch(messages_per_partition) unless processor&.rebalancing_or_shutting_down?
+              end
+            when :single then
+              message = consumer.poll(config.max_wait_time_ms)
+              if message
+                processor = assign_and_get_processor(message)
+                processor&.process(message) unless processor&.rebalancing_or_shutting_down?
+              end
             end
-          when :single then
-            message = consumer.poll(config.max_wait_time_ms)
-            process(message) if message
           end
         end
+      ensure
+        logger.info "Gracefully shutting down"
+        shutdown_processors_and_wait
+        consumer.commit
       end
-
-      logger.info "Gracefully shutting down"
+    ensure
       begin
-        processor.deliver!
-        processor.teardown
-        consumer.commit
-      ensure
         @instrumenter.instrument('leave_group') do
           consumer.close
         end
+      ensure
+        Racecar::Datadog.close if config.datadog_enabled
+        @instrumenter.instrument("shut_down", loop_payload || {})
       end
-    ensure
-      producer.close
-      Racecar::Datadog.close if config.datadog_enabled
-      @instrumenter.instrument("shut_down", instrumentation_payload || {})
     end
 
     def stop
       @stop_requested = true
     end
 
-    private
+    def consumer
+      @consumer ||= begin
+        ConsumerSet.new(config, logger, @partition_processors, @instrumenter)
+      end
+    end
 
-    attr_reader :pauses
+    private
 
     def process_method
       @process_method ||= begin
         case
-        when processor.respond_to?(:process_batch)
-          if processor.method(:process_batch).arity != 1
+        when consumer_class.method_defined?(:process_batch)
+          if consumer_class.instance_method(:process_batch).arity != 1
             raise Racecar::Error, "Invalid method signature for `process_batch`. The method must take exactly 1 argument."
           end
 
           :batch
-        when processor.respond_to?(:process)
-          if processor.method(:process).arity != 1
+        when consumer_class.method_defined?(:process)
+          if consumer_class.instance_method(:process).arity != 1
             raise Racecar::Error, "Invalid method signature for `process`. The method must take exactly 1 argument."
           end
 
           :single
         else
-          raise NotImplementedError, "Consumer class `#{processor.class}` must implement a `process` or `process_batch` method"
+          raise NotImplementedError, "Consumer class `#{consumer_class}` must implement a `process` or `process_batch` method"
         end
       end
     end
 
-    def consumer
-      @consumer ||= begin
-        ConsumerSet.new(config, logger, @instrumenter)
-      end
-    end
-
-    def producer
-      @producer ||= Rdkafka::Config.new(producer_config).producer.tap do |producer|
-        producer.delivery_callback = Racecar::DeliveryCallback.new(instrumenter: @instrumenter)
-      end
-    end
-
-    def producer_config
-      # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
-      producer_config = {
-        "bootstrap.servers"      => config.brokers.join(","),
-        "client.id"              => config.client_id,
-        "statistics.interval.ms" => config.statistics_interval_ms,
-        "message.timeout.ms"     => config.message_timeout * 1000,
-        "partitioner"            => config.partitioner.to_s,
-      }
-
-      producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
-      producer_config.merge!(config.rdkafka_producer)
-      producer_config
-    end
-
     def install_signal_handlers
       # Stop the consumer on SIGINT, SIGQUIT or SIGTERM.
       trap("QUIT") { stop }
@@ -166,128 +130,65 @@ module Racecar
       trap("USR1") { $stderr.puts config.inspect }
     end
 
-    def process(message)
-      instrumentation_payload = {
-        consumer_class: processor.class.to_s,
-        topic:          message.topic,
-        partition:      message.partition,
-        offset:         message.offset,
-        create_time:    message.timestamp,
-        key:            message.key,
-        value:          message.payload,
-        headers:        message.headers
-      }
-
-      @instrumenter.instrument("start_process_message", instrumentation_payload)
-      with_pause(message.topic, message.partition, message.offset..message.offset) do |pause|
-        begin
-          @instrumenter.instrument("process_message", instrumentation_payload) do
-            processor.process(Racecar::Message.new(message, retries_count: pause.pauses_count))
-            processor.deliver!
-            consumer.store_offset(message)
-          end
-        rescue => e
-          instrumentation_payload[:unrecoverable_delivery_error] = reset_producer_on_unrecoverable_delivery_errors(e)
-          instrumentation_payload[:retries_count] = pause.pauses_count
-          config.error_handler.call(e, instrumentation_payload)
-          raise e
-        end
+    def assign_and_get_processor(messages)
+      topic     = messages.is_a?(Array) ? messages.first.topic     : messages.topic
+      partition = messages.is_a?(Array) ? messages.first.partition : messages.partition
+      key = Runner.topic_partition_key(topic, partition)
+      return partition_processors[key] if partition_processors[key]
+
+      processor = if config.multithreaded_processing_enabled
+        AsyncPartitionProcessor.new(
+          **common_processor_params,
+          consumer_class: consumer_class,
+          topic: topic,
+          partition: partition,
+          rdkafka_consumer: consumer.current,
+        )
+      else
+        PartitionProcessor.new(
+          **common_processor_params,
+          consumer_class_instance: @consumer_class_instance,
+          topic: topic,
+          partition: partition,
+          pause: Pause.new_from_config(config),
+        )
       end
+      partition_processors[key] = processor
     end
 
-    def process_batch(messages)
-      first, last = messages.first, messages.last
-      instrumentation_payload = {
-        consumer_class: processor.class.to_s,
-        topic:          first.topic,
-        partition:      first.partition,
-        first_offset:   first.offset,
-        last_offset:    last.offset,
-        last_create_time: last.timestamp,
-        message_count:  messages.size
-      }
-
-      @instrumenter.instrument("start_process_batch", instrumentation_payload)
-      with_pause(first.topic, first.partition, first.offset..last.offset) do |pause|
-        begin
-          @instrumenter.instrument("process_batch", instrumentation_payload) do
-            racecar_messages = messages.map do |message|
-              Racecar::Message.new(message, retries_count: pause.pauses_count)
+    def shutdown_processors_and_wait
+      if config.multithreaded_processing_enabled
+        processors_snapshot = partition_processors.values
+        processors_snapshot.each { |processor| processor.shut_down! if processor }
+        processors_snapshot.each do |processor|
+          if processor.respond_to?(:thread)
+            begin
+              processor.thread.join(config.multithreaded_processing_shutdown_timeout)
+            rescue => e
+              logger.error "Error while waiting for processor thread to finish: #{e}"
             end
-            processor.process_batch(racecar_messages)
-            processor.deliver!
-            consumer.store_offset(messages.last)
           end
-        rescue => e
-          instrumentation_payload[:unrecoverable_delivery_error] = reset_producer_on_unrecoverable_delivery_errors(e)
-          instrumentation_payload[:retries_count] = pause.pauses_count
-          config.error_handler.call(e, instrumentation_payload)
-          raise e
+        end
+      else
+        begin
+          @consumer_class_instance.deliver!
+        ensure
+          @consumer_class_instance.teardown
         end
       end
     end
 
-    # librdkafka will continue to try to deliver already queued messages, even if ruby-rdkafka
-    # raised before that. This method detects any unrecoverable errors and resets the producer
-    # as a last ditch effort.
-    # The function returns true if there were unrecoverable errors, or false otherwise.
-    def reset_producer_on_unrecoverable_delivery_errors(error)
-      return false unless error.is_a?(Racecar::MessageDeliveryError)
-      return false unless error.code == :msg_timed_out # -192
-
-      logger.error error.to_s
-      logger.error "Racecar will reset the producer to force a new broker connection."
-      @producer.close
-      @producer = nil
-      processor.configure(
-        producer:     producer,
-        consumer:     consumer,
-        instrumenter: @instrumenter,
-        config:       @config,
-      )
-
-      true
-    end
-
-    def with_pause(topic, partition, offsets)
-      pause = pauses[topic][partition]
-      return yield pause if config.pause_timeout == 0
-
-      begin
-        yield pause
-        # We've successfully processed a batch from the partition, so we can clear the pause.
-        pauses[topic][partition].reset!
-      rescue => e
-        desc = "#{topic}/#{partition}"
-        logger.error "Failed to process #{desc} at #{offsets}: #{e}"
-
-        logger.warn "Pausing partition #{desc} for #{pause.backoff_interval} seconds"
-        consumer.pause(topic, partition, offsets.first)
-        pause.pause!
-      end
+    def resume_all_paused_partitions
+      partition_processors.values.reject(&:rebalancing_or_shutting_down?).each(&:resume_paused_partition)
     end
 
-    def resume_paused_partitions
-      return if config.pause_timeout == 0
-
-      pauses.each do |topic, partitions|
-        partitions.each do |partition, pause|
-          instrumentation_payload = {
-            topic:      topic,
-            partition:  partition,
-            duration:   pause.pause_duration,
-            consumer_class: processor.class.to_s,
-          }
-          @instrumenter.instrument("pause_status", instrumentation_payload)
-
-          if pause.paused? && pause.expired?
-            logger.info "Automatically resuming partition #{topic}/#{partition}, pause timeout expired"
-            consumer.resume(topic, partition)
-            pause.resume!
-            # TODO: # During re-balancing we might have lost the paused partition. Check if partition is still in group before seek. ?
-          end
-        end
-      end
+    def common_processor_params
+      {
+        config: config,
+        logger: logger,
+        instrumenter: @instrumenter,
+        consumer: consumer,
+      }
     end
   end
 end

data/lib/racecar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Racecar
-  VERSION = "2.12.0"
+  VERSION = "3.0.0.alpha.1"
 end

data/lib/racecar.rb

@@ -65,12 +65,12 @@ module Racecar
     config.instrumenter
   end
 
-  def self.run(processor)
-    runner(processor).run
+  def self.run(consumer_class)
+    runner(consumer_class).run
   end
 
-  def self.runner(processor)
-    runner = Runner.new(processor, config: config, logger: logger, instrumenter: config.instrumenter)
+  def self.runner(consumer_class)
+    runner = Runner.new(consumer_class, config: config, logger: logger, instrumenter: config.instrumenter)
 
     if config.parallel_workers && config.parallel_workers > 1
       ParallelRunner.new(runner: runner, config: config, logger: logger)

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.12.0
+  version: 3.0.0.alpha.1
 platform: ruby
 authors:
 - Daniel Schierbeck
 - Benjamin Quorning
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-02-27 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -149,7 +148,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
 email:
 - dschierbeck@zendesk.com
 - bquorning@zendesk.com
@@ -159,6 +157,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
 - ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
 - ".github/workflows/publish.yml"
@@ -187,6 +186,7 @@ files:
 - lib/generators/templates/consumer.rb.erb
 - lib/generators/templates/racecar.yml.erb
 - lib/racecar.rb
+- lib/racecar/async_partition_processor.rb
 - lib/racecar/cli.rb
 - lib/racecar/config.rb
 - lib/racecar/consumer.rb
@@ -203,6 +203,7 @@ files:
 - lib/racecar/message_delivery_error.rb
 - lib/racecar/null_instrumenter.rb
 - lib/racecar/parallel_runner.rb
+- lib/racecar/partition_processor.rb
 - lib/racecar/pause.rb
 - lib/racecar/producer.rb
 - lib/racecar/rails_config_file_loader.rb
@@ -214,7 +215,6 @@ homepage: https://github.com/zendesk/racecar
 licenses:
 - Apache License Version 2.0
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -229,8 +229,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: A framework for running Kafka consumers
 test_files: []