racecar 2.0.0 → 2.10.0.beta2

data/lib/racecar/runner.rb

@@ -1,6 +1,11 @@
+# frozen_string_literal: true
+
 require "rdkafka"
 require "racecar/pause"
 require "racecar/message"
+require "racecar/message_delivery_error"
+require "racecar/erroneous_state_error"
+require "racecar/delivery_callback"
 
 module Racecar
   class Runner
@@ -51,6 +56,7 @@ module Racecar
         producer:     producer,
         consumer:     consumer,
         instrumenter: @instrumenter,
+        config:       @config,
       )
 
       instrumentation_payload = {
@@ -62,6 +68,8 @@ module Racecar
       loop do
         break if @stop_requested
         resume_paused_partitions
+
+        @instrumenter.instrument("start_main_loop", instrumentation_payload)
         @instrumenter.instrument("main_loop", instrumentation_payload) do
           case process_method
           when :batch then
@@ -77,12 +85,19 @@ module Racecar
       end
 
       logger.info "Gracefully shutting down"
-      processor.deliver!
-      processor.teardown
-      consumer.commit
-      @instrumenter.instrument('leave_group') do
-        consumer.close
+      begin
+        processor.deliver!
+        processor.teardown
+        consumer.commit
+      ensure
+        @instrumenter.instrument('leave_group') do
+          consumer.close
+        end
       end
+    ensure
+      producer.close
+      Racecar::Datadog.close if config.datadog_enabled
+      @instrumenter.instrument("shut_down", instrumentation_payload || {})
     end
 
     def stop
@@ -96,28 +111,33 @@ module Racecar
     def process_method
       @process_method ||= begin
         case
-        when processor.respond_to?(:process_batch) then :batch
-        when processor.respond_to?(:process) then :single
+        when processor.respond_to?(:process_batch)
+          if processor.method(:process_batch).arity != 1
+            raise Racecar::Error, "Invalid method signature for `process_batch`. The method must take exactly 1 argument."
+          end
+
+          :batch
+        when processor.respond_to?(:process)
+          if processor.method(:process).arity != 1
+            raise Racecar::Error, "Invalid method signature for `process`. The method must take exactly 1 argument."
+          end
+
+          :single
         else
-          raise NotImplementedError, "Consumer class must implement process or process_batch method"
+          raise NotImplementedError, "Consumer class `#{processor.class}` must implement a `process` or `process_batch` method"
         end
       end
     end
 
     def consumer
       @consumer ||= begin
-        # Manually store offset after messages have been processed successfully
-        # to avoid marking failed messages as committed. The call just updates
-        # a value within librdkafka and is asynchronously written to proper
-        # storage through auto commits.
-        config.consumer << "enable.auto.offset.store=false"
         ConsumerSet.new(config, logger, @instrumenter)
       end
     end
 
     def producer
       @producer ||= Rdkafka::Config.new(producer_config).producer.tap do |producer|
-        producer.delivery_callback = delivery_callback
+        producer.delivery_callback = Racecar::DeliveryCallback.new(instrumenter: @instrumenter)
       end
     end
 
@@ -126,23 +146,16 @@ module Racecar
       producer_config = {
         "bootstrap.servers"      => config.brokers.join(","),
         "client.id"              => config.client_id,
-        "statistics.interval.ms" => 1000,
+        "statistics.interval.ms" => config.statistics_interval_ms,
+        "message.timeout.ms"     => config.message_timeout * 1000,
+        "partitioner"            => config.partitioner.to_s,
       }
+
       producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
       producer_config.merge!(config.rdkafka_producer)
       producer_config
     end
 
-    def delivery_callback
-      ->(delivery_report) do
-        payload = {
-          offset: delivery_report.offset,
-          partition: delivery_report.partition
-        }
-        @instrumenter.instrument("acknowledged_message", payload)
-      end
-    end
-
     def install_signal_handlers
       # Stop the consumer on SIGINT, SIGQUIT or SIGTERM.
       trap("QUIT") { stop }
@@ -166,14 +179,16 @@ module Racecar
       }
 
       @instrumenter.instrument("start_process_message", instrumentation_payload)
-      with_pause(message.topic, message.partition, message.offset..message.offset) do
+      with_pause(message.topic, message.partition, message.offset..message.offset) do |pause|
         begin
           @instrumenter.instrument("process_message", instrumentation_payload) do
-            processor.process(Racecar::Message.new(message))
+            processor.process(Racecar::Message.new(message, retries_count: pause.pauses_count))
             processor.deliver!
             consumer.store_offset(message)
           end
         rescue => e
+          instrumentation_payload[:unrecoverable_delivery_error] = reset_producer_on_unrecoverable_delivery_errors(e)
+          instrumentation_payload[:retries_count] = pause.pauses_count
           config.error_handler.call(e, instrumentation_payload)
           raise e
         end
@@ -193,32 +208,59 @@ module Racecar
       }
 
       @instrumenter.instrument("start_process_batch", instrumentation_payload)
-      @instrumenter.instrument("process_batch", instrumentation_payload) do
-        with_pause(first.topic, first.partition, first.offset..last.offset) do
-          begin
-            processor.process_batch(messages.map {|message| Racecar::Message.new(message) })
+      with_pause(first.topic, first.partition, first.offset..last.offset) do |pause|
+        begin
+          @instrumenter.instrument("process_batch", instrumentation_payload) do
+            racecar_messages = messages.map do |message|
+              Racecar::Message.new(message, retries_count: pause.pauses_count)
+            end
+            processor.process_batch(racecar_messages)
             processor.deliver!
             consumer.store_offset(messages.last)
-          rescue => e
-            config.error_handler.call(e, instrumentation_payload)
-            raise e
           end
+        rescue => e
+          instrumentation_payload[:unrecoverable_delivery_error] = reset_producer_on_unrecoverable_delivery_errors(e)
+          instrumentation_payload[:retries_count] = pause.pauses_count
+          config.error_handler.call(e, instrumentation_payload)
+          raise e
         end
       end
     end
 
+    # librdkafka will continue to try to deliver already queued messages, even if ruby-rdkafka
+    # raised before that. This method detects any unrecoverable errors and resets the producer
+    # as a last ditch effort.
+    # The function returns true if there were unrecoverable errors, or false otherwise.
+    def reset_producer_on_unrecoverable_delivery_errors(error)
+      return false unless error.is_a?(Racecar::MessageDeliveryError)
+      return false unless error.code == :msg_timed_out # -192
+
+      logger.error error.to_s
+      logger.error "Racecar will reset the producer to force a new broker connection."
+      @producer.close
+      @producer = nil
+      processor.configure(
+        producer:     producer,
+        consumer:     consumer,
+        instrumenter: @instrumenter,
+        config:       @config,
+      )
+
+      true
+    end
+
     def with_pause(topic, partition, offsets)
-      return yield if config.pause_timeout == 0
+      pause = pauses[topic][partition]
+      return yield pause if config.pause_timeout == 0
 
       begin
-        yield
+        yield pause
         # We've successfully processed a batch from the partition, so we can clear the pause.
         pauses[topic][partition].reset!
       rescue => e
         desc = "#{topic}/#{partition}"
         logger.error "Failed to process #{desc} at #{offsets}: #{e}"
 
-        pause = pauses[topic][partition]
         logger.warn "Pausing partition #{desc} for #{pause.backoff_interval} seconds"
         consumer.pause(topic, partition, offsets.first)
         pause.pause!

data/lib/racecar/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
-  VERSION = "2.0.0"
+  VERSION = "2.10.0.beta2"
 end

data/lib/racecar.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "logger"
 
 require "racecar/instrumenter"
@@ -5,7 +7,10 @@ require "racecar/null_instrumenter"
 require "racecar/consumer"
 require "racecar/consumer_set"
 require "racecar/runner"
+require "racecar/parallel_runner"
+require "racecar/producer"
 require "racecar/config"
+require "racecar/version"
 require "ensure_hash_compact"
 
 module Racecar
@@ -35,19 +40,42 @@ module Racecar
     config.logger = logger
   end
 
-  def self.instrumenter
-    @instrumenter ||= begin
-      default_payload = { client_id: config.client_id, group_id: config.group_id }
+  def self.produce_async(value:, topic:, **options)
+    producer.produce_async(value: value, topic: topic, **options)
+  end
 
-      Instrumenter.new(default_payload).tap do |instrumenter|
-        if instrumenter.backend == NullInstrumenter
-          logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
-        end
+  def self.produce_sync(value:, topic:, **options)
+    producer.produce_sync(value: value, topic: topic, **options)
+  end  
+  
+  def self.wait_for_delivery(&block)
+    producer.wait_for_delivery(&block)
+  end
+
+  def self.producer
+    Thread.current[:racecar_producer] ||= begin
+      if config.datadog_enabled
+        require "racecar/datadog"
       end
+      Racecar::Producer.new(config: config, logger: logger, instrumenter: instrumenter)
     end
   end
 
+  def self.instrumenter
+    config.instrumenter
+  end
+
   def self.run(processor)
-    Runner.new(processor, config: config, logger: logger, instrumenter: instrumenter).run
+    runner(processor).run
+  end
+
+  def self.runner(processor)
+    runner = Runner.new(processor, config: config, logger: logger, instrumenter: config.instrumenter)
+
+    if config.parallel_workers && config.parallel_workers > 1
+      ParallelRunner.new(runner: runner, config: config, logger: logger)
+    else
+      runner
+    end
   end
 end

data/racecar.gemspec

@@ -20,13 +20,16 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "king_konf", "~> 0.3.7"
-  spec.add_runtime_dependency "rdkafka",   "~> 0.8.0.beta.1"
+  spec.required_ruby_version = '>= 2.6'
+
+  spec.add_runtime_dependency "king_konf", "~> 1.0.0"
+  spec.add_runtime_dependency "rdkafka",   "~> 0.13.0"
 
   spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
+  spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "rake", "> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "timecop"
-  spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 5.0.0"
-  spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
+  spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 6.0.0"
+  spec.add_development_dependency "activesupport"
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.10.0.beta2
 platform: ruby
 authors:
 - Daniel Schierbeck
 - Benjamin Quorning
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-24 00:00:00.000000000 Z
+date: 2023-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -17,28 +17,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.7
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.7
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0.beta.1
+        version: 0.13.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0.beta.1
+        version: 0.13.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -59,6 +59,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -110,7 +124,7 @@ dependencies:
         version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 6.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -120,28 +134,22 @@ dependencies:
         version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 6.0.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.1'
-description:
+        version: '0'
+description: 
 email:
 - dschierbeck@zendesk.com
 - bquorning@zendesk.com
@@ -151,22 +159,28 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/rspec.yml"
+- ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
+- ".github/workflows/publish.yml"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md
+- Dockerfile
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - Procfile
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
+- docker-compose.yml
 - examples/batch_consumer.rb
 - examples/cat_consumer.rb
 - examples/producing_consumer.rb
 - exe/racecar
 - exe/racecarctl
+- extra/datadog-dashboard.json
 - lib/ensure_hash_compact.rb
 - lib/generators/racecar/consumer_generator.rb
 - lib/generators/racecar/install_generator.rb
@@ -180,11 +194,19 @@ files:
 - lib/racecar/ctl.rb
 - lib/racecar/daemon.rb
 - lib/racecar/datadog.rb
+- lib/racecar/delivery_callback.rb
+- lib/racecar/erroneous_state_error.rb
+- lib/racecar/heroku.rb
 - lib/racecar/instrumenter.rb
+- lib/racecar/liveness_probe.rb
 - lib/racecar/message.rb
+- lib/racecar/message_delivery_error.rb
 - lib/racecar/null_instrumenter.rb
+- lib/racecar/parallel_runner.rb
 - lib/racecar/pause.rb
+- lib/racecar/producer.rb
 - lib/racecar/rails_config_file_loader.rb
+- lib/racecar/rebalance_listener.rb
 - lib/racecar/runner.rb
 - lib/racecar/version.rb
 - racecar.gemspec
@@ -192,7 +214,7 @@ homepage: https://github.com/zendesk/racecar
 licenses:
 - Apache License Version 2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -200,15 +222,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
 summary: A framework for running Kafka consumers
 test_files: []

data/.github/workflows/rspec.yml

@@ -1,24 +0,0 @@
-name: Execute Specs
-
-on: [push]
-
-jobs:
-  rspec:
-
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        ruby-version: ["2.5.x", "2.6.x"]
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Ruby 2.6
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby-version }}
-    - name: Build and test with RSpec
-      run: |
-        gem install bundler --no-document
-        bundle install --jobs 4 --retry 3
-        bundle exec rspec --format documentation --require spec_helper --color