racecar 2.9.0.beta1 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 364239e804c99b816c0a37802b605ff4035d2d4c51abc0b161e11609268349ea
-  data.tar.gz: 4d7bd65a04b8c914640a2edc11f3c7521b2e061db8eb3c215ba29a785f031e12
+  metadata.gz: b394d31edcb9c83e562811ba748b6c6915388d0227a4e7425aa9aa13f64e3890
+  data.tar.gz: 1dfcd7046b6d932f716246d0685589c1635af115a4e126067c9775b13fe05464
 SHA512:
-  metadata.gz: c60037cd0dd5477a77a5fddd42003a3d4be67c38fb1c812cfff698cdb8fb7cd62e7e65097b6508c2a19a768daad5139f3eb469e91f0ef3ff3fe9535f9fd86a8a
-  data.tar.gz: 88d3ffcfa27b6bcb89b454151db353165ab62fa0dd42586c177d05ea6091663227b8d78f14180e69c4937417d0f761fc5666657be1320d44d54eeebcc5d78601
+  metadata.gz: 59043be21e411e680c11815b583236556413a539f5aa9508a460eefe82cee6199f56a30b9fa07a3f206886d5dc811b210cea6130813a7073a68fe6612343ca0d
+  data.tar.gz: d6692b9bb7cdc27efe5272a10fa1d9061920084a7b517699e5133f5e31e9fd50415e72ee40550e07e5a2fed028aeabb741422fef47372c7a53e0a96bdfc1b090

data/.github/workflows/ci.yml

@@ -12,30 +12,34 @@ jobs:
 
     strategy:
       matrix:
-        ruby-version: ["2.6", "3.0"]
+        ruby-version:
+          - "2.7"
+          - "3.0"
+          - "3.1"
+          - "3.2"
 
     steps:
-    - uses: zendesk/checkout@v2
+    - uses: zendesk/checkout@v3
     - name: Set up Ruby
-      uses: zendesk/setup-ruby@v1.64.1
+      uses: zendesk/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true
     - name: Build and test with RSpec
-      run: bundle exec rspec --format documentation --require spec_helper --color --exclude-pattern='spec/integration/*_spec.rb'
+      run: bundle exec rspec --exclude-pattern='spec/integration/*_spec.rb'
 
   integration-specs:
     runs-on: ubuntu-latest
     steps:
-    - uses: zendesk/checkout@v2
+    - uses: zendesk/checkout@v3
     - name: Set up Ruby
-      uses: zendesk/setup-ruby@v1.64.1
+      uses: zendesk/setup-ruby@v1
       with:
-        ruby-version: 2.7
+        ruby-version: "2.7"
         bundler-cache: true
     - name: Bring up docker-compose stack
       run: docker-compose up -d
     - name: Build and test with RSpec
       env:
         RACECAR_BROKERS: localhost:9092
-      run: timeout --kill-after 180 150 bundle exec rspec --format documentation --require spec_helper --color spec/integration/*_spec.rb
+      run: timeout --kill-after 180 150 bundle exec rspec spec/integration/*_spec.rb

data/CHANGELOG.md

@@ -1,6 +1,10 @@
 # Changelog
 
-## v2.9.0.beta1
+## Unreleased
+
+* Test with Ruby 3.2
+
+## v2.9.0, v2.9.0.beta1
 
 * Add `partitioner` producer config option to allow changing the strategy to
   determine which topic partition a message is written to when racecar

data/Gemfile

@@ -5,6 +5,6 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in racecar.gemspec
 gemspec
 
-# We actually support version 6.0 (see gemspec); this extra restriction is added just for running the test suite also
-# on Ruby 2.4, which activesupport 6.0 no longer supports
-gem 'activesupport', '< 6.0'
+# We actually support version 7.x (see gemspec); this extra restriction is added just for running the test suite also
+# on Ruby 2.6, which activesupport 7.0 does not support.
+gem 'activesupport', '~> 6.1.0'

data/Gemfile.lock

@@ -8,60 +8,65 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.6)
+    activesupport (6.1.7.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    byebug (11.1.3)
     coderay (1.1.3)
-    concurrent-ruby (1.1.9)
-    diff-lcs (1.4.4)
-    dogstatsd-ruby (5.2.0)
+    concurrent-ruby (1.2.2)
+    diff-lcs (1.5.0)
+    dogstatsd-ruby (5.5.0)
     ffi (1.15.5)
-    i18n (1.8.10)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     king_konf (1.0.1)
     method_source (1.0.0)
     mini_portile2 (2.8.1)
-    minitest (5.14.4)
-    pry (0.13.1)
+    minitest (5.18.0)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rake (13.0.1)
+    pry-byebug (3.10.1)
+      byebug (~> 11.0)
+      pry (>= 0.13, < 0.15)
+    rake (13.0.6)
     rdkafka (0.12.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.1)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.4)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    thread_safe (0.3.6)
-    timecop (0.9.2)
-    tzinfo (1.2.10)
-      thread_safe (~> 0.1)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    timecop (0.9.6)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.6.7)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport (< 6.0)
+  activesupport (~> 6.1.0)
   bundler (>= 1.13, < 3)
   dogstatsd-ruby (>= 4.0.0, < 6.0.0)
-  pry
+  pry-byebug
   racecar!
   rake (> 10.0)
   rspec (~> 3.0)
   timecop
 
 BUNDLED WITH
-   2.3.7
+   2.4.9

data/README.md

@@ -249,7 +249,8 @@ You can set message headers by passing a `headers:` option with a Hash of header
 ### Standalone Producer
 
 Racecar provides a standalone producer to publish messages to Kafka directly from your Rails application:
-```
+
+```ruby
 # app/controllers/comments_controller.rb
 class CommentsController < ApplicationController
   def create
@@ -263,7 +264,8 @@ end
 ```
 
 The above example will block the server process until the message has been delivered. If you want deliveries to happen in the background in order to free up your server processes more quickly, call #deliver_async instead:
-```
+
+```ruby
 # app/controllers/comments_controller.rb
 class CommentsController < ApplicationController
   def show
@@ -284,9 +286,9 @@ end
 ```
 In addition to improving response time, delivering messages asynchronously also protects your application against Kafka availability issues -- if messages cannot be delivered, they'll be buffered for later and retried automatically.
 
-A third method is to produce messages first (without delivering the messages to Kafka yet), and deliver them synchronously later.
+A third method is to produce messages first (without delivering the messages to Kafka yet), and deliver them synchronously later:
 
-```
+```ruby
  # app/controllers/comments_controller.rb
  class CommentsController < ApplicationController
    def create
@@ -415,7 +417,7 @@ Racecar supports [Datadog](https://www.datadoghq.com/) monitoring integration. I
 - `datadog_namespace` – The namespace to use for Datadog metrics.
 - `datadog_tags` – Tags that should always be set on Datadog metrics.
 
-Furthermore, there's a [standard Datadog dashboard configution file](https://raw.githubusercontent.com/zendesk/racecar/master/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
+Furthermore, there's a [standard Datadog dashboard configuration file](https://raw.githubusercontent.com/zendesk/racecar/master/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
 
 #### Consumers Without Rails
 

data/lib/racecar/cli.rb

@@ -19,6 +19,7 @@ module Racecar
       @parser = build_parser
       @parser.parse!(args)
       @consumer_name = args.first or raise Racecar::Error, "no consumer specified"
+      @runner = nil
     end
 
     def run
@@ -65,10 +66,15 @@ module Racecar
       end
 
       processor = consumer_class.new
-      Racecar.run(processor)
+      @runner = Racecar.runner(processor)
+      @runner.run
       nil
     end
 
+    def stop
+      @runner.stop
+    end
+
     private
 
     attr_reader :consumer_name

data/lib/racecar/config.rb

@@ -6,6 +6,7 @@ require "king_konf"
 
 require "racecar/liveness_probe"
 require "racecar/instrumenter"
+require "racecar/rebalance_listener"
 
 module Racecar
   class Config < KingKonf::Config
@@ -227,6 +228,7 @@ module Racecar
     end
 
     def load_consumer_class(consumer_class)
+      self.consumer_class = consumer_class
       self.group_id = consumer_class.group_id || self.group_id
 
       self.group_id ||= [
@@ -243,6 +245,7 @@ module Racecar
       self.fetch_messages = consumer_class.fetch_messages || self.fetch_messages
       self.pidfile ||= "#{group_id}.pid"
     end
+    attr_accessor :consumer_class
 
     def on_error(&handler)
       @error_handler = handler
@@ -293,6 +296,10 @@ module Racecar
       )
     end
 
+    def rebalance_listener
+      RebalanceListener.new(self)
+    end
+
     private
 
     def rdkafka_security_config

data/lib/racecar/consumer_set.rb

@@ -69,7 +69,10 @@ module Racecar
 
     def current
       @consumers[@consumer_id_iterator.peek] ||= begin
-        consumer = Rdkafka::Config.new(rdkafka_config(current_subscription)).consumer
+        consumer_config = Rdkafka::Config.new(rdkafka_config(current_subscription))
+        consumer_config.consumer_rebalance_listener = @config.rebalance_listener
+
+        consumer = consumer_config.consumer
         @instrumenter.instrument('join_group') do
           consumer.subscribe current_subscription.topic
         end

data/lib/racecar/daemon.rb

@@ -54,7 +54,7 @@ module Racecar
     end
 
     def pid
-      if File.exists?(pidfile)
+      if File.exist?(pidfile)
         File.read(pidfile).to_i
       else
         nil
@@ -89,7 +89,7 @@ module Racecar
       end
 
       at_exit do
-        File.delete(pidfile) if File.exists?(pidfile)
+        File.delete(pidfile) if File.exist?(pidfile)
       end
     rescue Errno::EEXIST
       check_pid

data/lib/racecar/datadog.rb

@@ -211,6 +211,10 @@ module Racecar
           topic: topic,
         }
 
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
         # This gets us the write rate.
         increment("producer.produce.messages", tags: tags.merge(topic: topic))
 
@@ -244,7 +248,15 @@ module Racecar
         # Number of messages ACK'd for the topic.
         increment("producer.ack.messages", tags: tags)
       end
-      
+
+      def produce_delivery_error(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+        }
+
+        increment("producer.produce.delivery.errors", tags: tags)
+      end
+
       def produce_async(event)
         client = event.payload.fetch(:client_id)
         topic = event.payload.fetch(:topic)
@@ -256,6 +268,10 @@ module Racecar
           topic: topic,
         }
 
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
         # This gets us the write rate.
         increment("producer.produce.messages", tags: tags.merge(topic: topic))
 
@@ -279,6 +295,11 @@ module Racecar
           topic: topic,
         }
 
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
+
         # This gets us the write rate.
         increment("producer.produce.messages", tags: tags.merge(topic: topic))
 

data/lib/racecar/delivery_callback.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Racecar
+  class DeliveryCallback
+    attr_reader :instrumenter
+
+    def initialize(instrumenter:)
+      @instrumenter = instrumenter
+    end
+
+    def call(delivery_report)
+      if delivery_report.error.to_i.zero?
+        payload = {
+          offset: delivery_report.offset,
+          partition: delivery_report.partition
+        }
+        instrumenter.instrument("acknowledged_message", payload)
+      else
+        payload = {
+          partition: delivery_report.partition,
+          exception: delivery_report.error
+        }
+        instrumenter.instrument("produce_delivery_error", payload)
+      end
+    end
+  end
+end

data/lib/racecar/parallel_runner.rb

@@ -37,6 +37,10 @@ module Racecar
       wait_for_exit
     end
 
+    def stop
+      terminate_workers
+    end
+
     private
 
     attr_accessor :workers

data/lib/racecar/producer.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "racecar/message_delivery_error"
+require "racecar/delivery_callback"
 
 at_exit do
   Racecar::Producer.shutdown!
@@ -42,19 +43,24 @@ module Racecar
           }
           producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
           producer_config.merge!(config.rdkafka_producer)
-          Rdkafka::Config.new(producer_config).producer
+          Rdkafka::Config.new(producer_config).producer.tap do |producer|
+            producer.delivery_callback = DeliveryCallback.new(instrumenter: @instrumenter)
+          end
         end
       end
     end
 
-
-    # fire and forget - you won't get any guarantees or feedback from 
+    # fire and forget - you won't get any guarantees or feedback from
     # Racecar on the status of the message and it won't halt execution
     # of the rest of your code.
     def produce_async(value:, topic:, **options)
       with_instrumentation(action: "produce_async", value: value, topic: topic, **options) do
-        handle = internal_producer.produce(payload: value, topic: topic, **options)
-        @delivery_handles << handle if @batching
+        begin
+          handle = internal_producer.produce(payload: value, topic: topic, **options)
+          @delivery_handles << handle if @batching
+        rescue Rdkafka::RdkafkaError => e
+          raise MessageDeliveryError.new(e, handle)
+        end
       end
 
       nil
@@ -63,8 +69,12 @@ module Racecar
     # synchronous message production - will wait until the delivery handle succeeds, fails or times out.
     def produce_sync(value:, topic:, **options)
       with_instrumentation(action: "produce_sync", value: value, topic: topic, **options) do
-        handle = internal_producer.produce(payload: value, topic: topic, **options)
-        deliver_with_error_handling(handle)
+        begin
+          handle = internal_producer.produce(payload: value, topic: topic, **options)
+          deliver_with_error_handling(handle)
+        rescue Rdkafka::RdkafkaError => e
+          raise MessageDeliveryError.new(e, handle)
+        end
       end
 
       nil
@@ -126,4 +136,4 @@ module Racecar
       end
     end
   end
-end
+end

data/lib/racecar/rebalance_listener.rb

@@ -0,0 +1,22 @@
+module Racecar
+  class RebalanceListener
+    def initialize(config)
+      @config = config
+      @consumer_class = config.consumer_class
+    end
+
+    attr_reader :config, :consumer_class
+
+    def on_partitions_assigned(_consumer, topic_partition_list)
+      consumer_class.respond_to?(:on_partitions_assigned) &&
+        consumer_class.on_partitions_assigned(topic_partition_list.to_h)
+    rescue
+    end
+
+    def on_partitions_revoked(_consumer, topic_partition_list)
+      consumer_class.respond_to?(:on_partitions_revoked) &&
+        consumer_class.on_partitions_revoked(topic_partition_list.to_h)
+    rescue
+    end
+  end
+end

data/lib/racecar/runner.rb

@@ -5,6 +5,7 @@ require "racecar/pause"
 require "racecar/message"
 require "racecar/message_delivery_error"
 require "racecar/erroneous_state_error"
+require "racecar/delivery_callback"
 
 module Racecar
   class Runner
@@ -141,7 +142,7 @@ module Racecar
 
     def producer
       @producer ||= Rdkafka::Config.new(producer_config).producer.tap do |producer|
-        producer.delivery_callback = delivery_callback
+        producer.delivery_callback = Racecar::DeliveryCallback.new(instrumenter: @instrumenter)
       end
     end
 
@@ -160,16 +161,6 @@ module Racecar
       producer_config
     end
 
-    def delivery_callback
-      ->(delivery_report) do
-        payload = {
-          offset: delivery_report.offset,
-          partition: delivery_report.partition
-        }
-        @instrumenter.instrument("acknowledged_message", payload)
-      end
-    end
-
     def install_signal_handlers
       # Stop the consumer on SIGINT, SIGQUIT or SIGTERM.
       trap("QUIT") { stop }

data/lib/racecar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Racecar
-  VERSION = "2.9.0.beta1"
+  VERSION = "2.9.0"
 end

data/lib/racecar.rb

@@ -66,12 +66,16 @@ module Racecar
   end
 
   def self.run(processor)
+    runner(processor).run
+  end
+
+  def self.runner(processor)
     runner = Runner.new(processor, config: config, logger: logger, instrumenter: config.instrumenter)
 
     if config.parallel_workers && config.parallel_workers > 1
-      ParallelRunner.new(runner: runner, config: config, logger: logger).run
+      ParallelRunner.new(runner: runner, config: config, logger: logger)
     else
-      runner.run
+      runner
     end
   end
 end

data/racecar.gemspec

@@ -26,10 +26,10 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "rdkafka",   "~> 0.12.0"
 
   spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
-  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "rake", "> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 6.0.0"
-  spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
+  spec.add_development_dependency "activesupport"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.9.0.beta1
+  version: 2.9.0
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-03-22 00:00:00.000000000 Z
+date: 2023-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -60,7 +60,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -141,20 +141,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '0'
 description: 
 email:
 - dschierbeck@zendesk.com
@@ -200,6 +194,7 @@ files:
 - lib/racecar/ctl.rb
 - lib/racecar/daemon.rb
 - lib/racecar/datadog.rb
+- lib/racecar/delivery_callback.rb
 - lib/racecar/erroneous_state_error.rb
 - lib/racecar/heroku.rb
 - lib/racecar/instrumenter.rb
@@ -211,6 +206,7 @@ files:
 - lib/racecar/pause.rb
 - lib/racecar/producer.rb
 - lib/racecar/rails_config_file_loader.rb
+- lib/racecar/rebalance_listener.rb
 - lib/racecar/runner.rb
 - lib/racecar/version.rb
 - racecar.gemspec
@@ -229,9 +225,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.3.1
 signing_key: