racecar 2.8.2 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4cbf6ca4837bdd3893c8342936f9d5ea115a5b3a585e0c843e62171e00038e72
-  data.tar.gz: d8603c46cf801d0f41f017d5b27b35e0d2b195eec5be11a1f7b6bf77413726b4
+  metadata.gz: b394d31edcb9c83e562811ba748b6c6915388d0227a4e7425aa9aa13f64e3890
+  data.tar.gz: 1dfcd7046b6d932f716246d0685589c1635af115a4e126067c9775b13fe05464
 SHA512:
-  metadata.gz: 78bdedfe7bd888a3716f5a29f28810f4c4d2f03e43c33b8e47ed08607155211fca1d47f4eab6a4f0def6976bb7ca3263117e52d2e0fa7f0a721c56ec70c7ee3d
-  data.tar.gz: 2bf6e2b50980add68363b59718f54403caada58abcb404a9018ac4304519bbbb7ed920578d37bcadefad63964812e7374d367298f403577725eef79af269c604
+  metadata.gz: 59043be21e411e680c11815b583236556413a539f5aa9508a460eefe82cee6199f56a30b9fa07a3f206886d5dc811b210cea6130813a7073a68fe6612343ca0d
+  data.tar.gz: d6692b9bb7cdc27efe5272a10fa1d9061920084a7b517699e5133f5e31e9fd50415e72ee40550e07e5a2fed028aeabb741422fef47372c7a53e0a96bdfc1b090

data/.github/workflows/ci.yml

@@ -12,30 +12,34 @@ jobs:
 
     strategy:
       matrix:
-        ruby-version: ["2.6", "3.0"]
+        ruby-version:
+          - "2.7"
+          - "3.0"
+          - "3.1"
+          - "3.2"
 
     steps:
-    - uses: zendesk/checkout@v2
+    - uses: zendesk/checkout@v3
     - name: Set up Ruby
-      uses: zendesk/setup-ruby@v1.64.1
+      uses: zendesk/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true
     - name: Build and test with RSpec
-      run: bundle exec rspec --format documentation --require spec_helper --color --exclude-pattern='spec/integration/*_spec.rb'
+      run: bundle exec rspec --exclude-pattern='spec/integration/*_spec.rb'
 
   integration-specs:
     runs-on: ubuntu-latest
     steps:
-    - uses: zendesk/checkout@v2
+    - uses: zendesk/checkout@v3
     - name: Set up Ruby
-      uses: zendesk/setup-ruby@v1.64.1
+      uses: zendesk/setup-ruby@v1
       with:
-        ruby-version: 2.7
+        ruby-version: "2.7"
         bundler-cache: true
     - name: Bring up docker-compose stack
       run: docker-compose up -d
     - name: Build and test with RSpec
       env:
         RACECAR_BROKERS: localhost:9092
-      run: timeout --kill-after 180 150 bundle exec rspec --format documentation --require spec_helper --color spec/integration/*_spec.rb
+      run: timeout --kill-after 180 150 bundle exec rspec spec/integration/*_spec.rb

data/.github/workflows/publish.yml

@@ -0,0 +1,12 @@
+name: Publish Gem
+
+on:
+  push:
+    tags: v*
+
+jobs:
+  call-workflow:
+    uses: zendesk/gw/.github/workflows/ruby-gem-publication.yml@main
+    secrets:
+      RUBY_GEMS_API_KEY: ${{ secrets.RUBY_GEMS_API_KEY }}
+      RUBY_GEMS_TOTP_DEVICE: ${{ secrets.RUBY_GEMS_TOTP_DEVICE }}

data/CHANGELOG.md

@@ -2,14 +2,23 @@
 
 ## Unreleased
 
+* Test with Ruby 3.2
+
+## v2.9.0, v2.9.0.beta1
+
+* Add `partitioner` producer config option to allow changing the strategy to
+  determine which topic partition a message is written to when racecar
+  produces a kafka message
+* Add built-in liveness probe for Kubernetes deployments.
+
 ## v2.8.2
-* Handles ErroneousStateError, in previous versions the consumer would do several unecessary group leave/joins. The log level is also changed to WARN instead of ERROR. ([#285](https://github.com/zendesk/racecar/pull/295))
+* Handles ErroneousStateError, in previous versions the consumer would do several unecessary group leave/joins. The log level is also changed to WARN instead of ERROR. ([#295](https://github.com/zendesk/racecar/pull/295))
 
 ## v2.8.1
 * Adds new ErroneousStateError to racecar in order to give more information on this new possible exception.
 
 ## v2.8.0
-* Update librdkafka version from 1.8.2 to 1.9.0 by upgrading from rdkafka 0.10.0 to 0.12.0. ([#283](https://github.com/zendesk/racecar/pull/293))
+* Update librdkafka version from 1.8.2 to 1.9.0 by upgrading from rdkafka 0.10.0 to 0.12.0. ([#293](https://github.com/zendesk/racecar/pull/293))
 
 ## v2.7.0
 

data/Gemfile

@@ -5,6 +5,6 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in racecar.gemspec
 gemspec
 
-# We actually support version 6.0 (see gemspec); this extra restriction is added just for running the test suite also
-# on Ruby 2.4, which activesupport 6.0 no longer supports
-gem 'activesupport', '< 6.0'
+# We actually support version 7.x (see gemspec); this extra restriction is added just for running the test suite also
+# on Ruby 2.6, which activesupport 7.0 does not support.
+gem 'activesupport', '~> 6.1.0'

data/Gemfile.lock

@@ -1,67 +1,72 @@
 PATH
   remote: .
   specs:
-    racecar (2.8.2.beta)
+    racecar (2.9.0.beta1)
       king_konf (~> 1.0.0)
       rdkafka (~> 0.12.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.6)
+    activesupport (6.1.7.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    byebug (11.1.3)
     coderay (1.1.3)
-    concurrent-ruby (1.1.9)
-    diff-lcs (1.4.4)
-    dogstatsd-ruby (5.2.0)
+    concurrent-ruby (1.2.2)
+    diff-lcs (1.5.0)
+    dogstatsd-ruby (5.5.0)
     ffi (1.15.5)
-    i18n (1.8.10)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    king_konf (1.0.0)
+    king_konf (1.0.1)
     method_source (1.0.0)
-    mini_portile2 (2.8.0)
-    minitest (5.14.4)
-    pry (0.13.1)
+    mini_portile2 (2.8.1)
+    minitest (5.18.0)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rake (13.0.1)
+    pry-byebug (3.10.1)
+      byebug (~> 11.0)
+      pry (>= 0.13, < 0.15)
+    rake (13.0.6)
     rdkafka (0.12.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.1)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.4)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    thread_safe (0.3.6)
-    timecop (0.9.2)
-    tzinfo (1.2.9)
-      thread_safe (~> 0.1)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    timecop (0.9.6)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.6.7)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport (< 6.0)
+  activesupport (~> 6.1.0)
   bundler (>= 1.13, < 3)
   dogstatsd-ruby (>= 4.0.0, < 6.0.0)
-  pry
+  pry-byebug
   racecar!
   rake (> 10.0)
   rspec (~> 3.0)
   timecop
 
 BUNDLED WITH
-   2.3.7
+   2.4.9

data/README.md

@@ -20,6 +20,7 @@ The framework is based on [rdkafka-ruby](https://github.com/appsignal/rdkafka-ru
    8. [Logging](#logging)
    9. [Operations](#operations)
    10. [Upgrading from v1 to v2](#upgrading-from-v1-to-v2)
+   11. [Compression](#compression)
 3. [Development](#development)
 4. [Contributing](#contributing)
 5. [Support and Discussion](#support-and-discussion)
@@ -245,6 +246,71 @@ The `deliver!` method can be used to block until the broker received all queued
 
 You can set message headers by passing a `headers:` option with a Hash of headers.
 
+### Standalone Producer
+
+Racecar provides a standalone producer to publish messages to Kafka directly from your Rails application:
+
+```ruby
+# app/controllers/comments_controller.rb
+class CommentsController < ApplicationController
+  def create
+    @comment = Comment.create!(params)
+
+    # This will publish a JSON representation of the comment to the `comments` topic
+    # in Kafka. Make sure to create the topic first, or this may fail.
+    Racecar.produce_sync(value:comment.to_json, topic: "comments")
+  end
+end
+```
+
+The above example will block the server process until the message has been delivered. If you want deliveries to happen in the background in order to free up your server processes more quickly, call #deliver_async instead:
+
+```ruby
+# app/controllers/comments_controller.rb
+class CommentsController < ApplicationController
+  def show
+    @comment = Comment.find(params[:id])
+
+    event = {
+      name: "comment_viewed",
+      data: {
+        comment_id: @comment.id,
+        user_id: current_user.id
+      }
+    }
+
+    # By delivering messages asynchronously you free up your server processes faster.
+    Racecar.produce_async(value: event.to_json, topic: "activity")
+  end
+end
+```
+In addition to improving response time, delivering messages asynchronously also protects your application against Kafka availability issues -- if messages cannot be delivered, they'll be buffered for later and retried automatically.
+
+A third method is to produce messages first (without delivering the messages to Kafka yet), and deliver them synchronously later:
+
+```ruby
+ # app/controllers/comments_controller.rb
+ class CommentsController < ApplicationController
+   def create
+     @comment = Comment.create!(params)
+
+     event = {
+       name: "comment_created",
+       data: {
+         comment_id: @comment.id
+         user_id: current_user.id
+       }
+     }
+
+     # This will queue the two messages in the internal buffer and block server process until they are delivered.
+    Racecar.wait_for_delivery do
+      Racecar.produce_async(comment.to_json, topic: "comments")
+      Racecar.produce_async(event.to_json, topic: "activity")
+    end
+   end
+ end
+ ```
+
 ### Configuration
 
 Racecar provides a flexible way to configure your consumer in a way that feels at home in a Rails application. If you haven't already, run `bundle exec rails generate racecar:install` in order to generate a config file. You'll get a separate section for each Rails environment, with the common configuration values in a shared `common` section.
@@ -338,6 +404,7 @@ Racecar has support for using SASL to authenticate clients using either the GSSA
 
 These settings are related to consumers that _produce messages to Kafka_.
 
+- `partitioner` – The strategy used to determine which topic partition a message is written to when Racecar produces a value to Kafka. The codec needs to be one of `consistent`, `consistent_random` `murmur2` `murmur2_random` `fnv1a` `fnv1a_random` either as a Symbol or a String, defaults to `consistent_random`
 - `producer_compression_codec` – If defined, Racecar will compress messages before writing them to Kafka. The codec needs to be one of `gzip`, `lz4`, or `snappy`, either as a Symbol or a String.
 
 #### Datadog monitoring
@@ -350,7 +417,7 @@ Racecar supports [Datadog](https://www.datadoghq.com/) monitoring integration. I
 - `datadog_namespace` – The namespace to use for Datadog metrics.
 - `datadog_tags` – Tags that should always be set on Datadog metrics.
 
-Furthermore, there's a [standard Datadog dashboard configution file](https://raw.githubusercontent.com/zendesk/racecar/master/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
+Furthermore, there's a [standard Datadog dashboard configuration file](https://raw.githubusercontent.com/zendesk/racecar/master/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
 
 #### Consumers Without Rails
 
@@ -443,6 +510,64 @@ The important part is the `strategy.type` value, which tells Kubernetes how to u
 
 Instead, the `Recreate` update strategy should be used. It completely tears down the existing containers before starting all of the new containers simultaneously, allowing for a single synchronization stage and a much faster, more stable deployment update.
 
+#### Liveness Probe
+
+Racecar comes with a built-in liveness probe, primarily for use with Kubernetes, but useful for any deployment environment where you can periodically run a process to check the health of your consumer.
+
+To use this feature:
+- set the `liveness_probe_enabled` config option to true.
+- configure your Kubernetes deployment to run `$ racecarctl liveness_probe`
+
+
+When enabled (see config) Racecar will touch the file at `liveness_probe_file_path` each time it finishes polling Kafka and processing the messages in the batch (if any).
+
+The modified time of this file can be observed to determine when the consumer last exhibited 'liveness'.
+
+Running `racecarctl liveness_probe` will return a successful exit status if the last 'liveness' event happened within an acceptable time, `liveness_probe_max_interval`.
+
+`liveness_probe_max_interval` should be long enough to account for both the Kafka polling time of `max_wait_time` and the processing time of a full message batch.
+
+On receiving `SIGTERM`, Racecar will gracefully shut down and delete this file, causing the probe to fail immediately after exit.
+
+You may wish to tolerate more than one failed probe run to accommodate for environmental variance and clock changes.
+
+See the [Configuration section](https://github.com/zendesk/racecar#configuration) for the various ways the liveness probe can be configured, environment variables being one option.
+
+Here is an example Kubernetes liveness probe configuration:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+spec:
+  template:
+    spec:
+      containers:
+      - name: consumer
+
+        args:
+        - racecar
+        - SomeConsumer
+
+        env:
+        - name: RACECAR_LIVENESS_PROBE_ENABLED
+          value: "true"
+
+        livenessProbe:
+          exec:
+            command:
+            - racecarctl
+            - liveness_probe
+
+          # Allow up to 10 consecutive failures before terminating Pod:
+          failureThreshold: 10
+
+          # Wait 30 seconds before starting the probes:
+          initialDelaySeconds: 30
+
+          # Perform the check every 10 seconds:
+          periodSeconds: 10
+```
+
 #### Deploying to Heroku
 
 If you run your applications in Heroku and/or use the Heroku Kafka add-on, you application will be provided with 4 ENV variables that allow connecting to the cluster: `KAFKA_URL`, `KAFKA_TRUSTED_CERT`, `KAFKA_CLIENT_CERT`, and `KAFKA_CLIENT_CERT_KEY`.
@@ -479,7 +604,7 @@ Again, the recommended approach is to manage the processes using process manager
 
 ### Handling errors
 
-When processing messages from a Kafka topic, your code may encounter an error and raise an exception. The cause is typically one of two things:
+#### When processing messages from a Kafka topic, your code may encounter an error and raise an exception. The cause is typically one of two things:
 
 1. The message being processed is somehow malformed or doesn't conform with the assumptions made by the processing code.
 2. You're using some external resource such as a database or a network API that is temporarily unavailable.
@@ -514,6 +639,16 @@ end
 
 It is highly recommended that you set up an error handler. Please note that the `info` object contains different keys and values depending on whether you are using `process` or `process_batch`. See the `instrumentation_payload` object in the `process` and `process_batch` methods in the `Runner` class for the complete list.
 
+#### Errors related to Compression
+
+A sample error might look like this:
+
+```
+E, [2022-10-09T11:28:29.976548 #15] ERROR -- : (try 5/10): Error for topic subscription #<struct Racecar::Consumer::Subscription topic="support.entity_incremental.views.view_ticket_ids", start_from_beginning=false, max_bytes_per_partition=104857, additional_config={}>: Local: Not implemented (not_implemented)
+```
+
+Please see [Compression](#compression)
+
 ### Logging
 
 By default, Racecar will log to `STDOUT`. If you're using Rails, your application code will use whatever logger you've configured there.
@@ -528,7 +663,15 @@ In order to introspect the configuration of a consumer process, send it the `SIG
 
 ### Upgrading from v1 to v2
 
-In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency. In general, you should avoid rolling deploys for consumers groups, so it is likely the case that this will just work for you, but it's a good idea to check first.
+In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency. In general, you should avoid rolling deploys for consumers groups, so it is likely the case that this will just work for you, but it's a good idea to check first. 
+
+### Compression
+
+Racecar v2 requires a C library (zlib) to compress the messages before producing to the topic. If not already installed on you consumer docker container, please install using following command in Dockerfile of consumer
+
+``` 
+apt-get update && apt-get install -y libzstd-dev
+```
 
 ## Development
 

data/Rakefile

@@ -3,6 +3,9 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 
+# Pushing to rubygems is handled by a github workflow
+ENV["gem_push"] = "false"
+
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec

data/examples/batch_consumer.rb

@@ -3,8 +3,8 @@
 class BatchConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 
-  def process_batch(batch)
-    batch.messages.each do |message|
+  def process_batch(messages)
+    messages.each do |message|
       puts message.value
     end
   end

data/lib/racecar/cli.rb

@@ -5,6 +5,7 @@ require "logger"
 require "fileutils"
 require "racecar/rails_config_file_loader"
 require "racecar/daemon"
+require "racecar/liveness_probe"
 
 module Racecar
   class Cli
@@ -18,6 +19,7 @@ module Racecar
       @parser = build_parser
       @parser.parse!(args)
       @consumer_name = args.first or raise Racecar::Error, "no consumer specified"
+      @runner = nil
     end
 
     def run
@@ -58,11 +60,21 @@ module Racecar
         $stderr.puts "=> Ctrl-C to shutdown consumer"
       end
 
+      if config.liveness_probe_enabled
+        $stderr.puts "=> Liveness probe enabled"
+        config.install_liveness_probe
+      end
+
       processor = consumer_class.new
-      Racecar.run(processor)
+      @runner = Racecar.runner(processor)
+      @runner.run
       nil
     end
 
+    def stop
+      @runner.stop
+    end
+
     private
 
     attr_reader :consumer_name

data/lib/racecar/config.rb

@@ -1,7 +1,13 @@
 # frozen_string_literal: true
 
+require "tmpdir"
+
 require "king_konf"
 
+require "racecar/liveness_probe"
+require "racecar/instrumenter"
+require "racecar/rebalance_listener"
+
 module Racecar
   class Config < KingKonf::Config
     env_prefix :racecar
@@ -77,6 +83,9 @@ module Racecar
     desc "The log level for the Racecar logs"
     string :log_level, default: "info"
 
+    desc "The strategy used to determine which topic partition a message is written to when Racecar produces a value to Kafka; defaults to `consistent_random`"
+    symbol :partitioner, allowed_values: %i{consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random}, default: :consistent_random
+
     desc "Protocol used to communicate with brokers"
     symbol :security_protocol, allowed_values: %i{plaintext ssl sasl_plaintext sasl_ssl}
 
@@ -164,6 +173,15 @@ module Racecar
           for backward compatibility, however this can be quite memory intensive"
     integer :statistics_interval, default: 1
 
+    desc "Whether to enable liveness probe behavior (touch the file)"
+    boolean :liveness_probe_enabled, default: false
+
+    desc "Path to a file Racecar will touch to show liveness"
+    string :liveness_probe_file_path, default: "#{Dir.tmpdir}/racecar-liveness"
+
+    desc "Used only by the liveness probe: Max time (in seconds) between liveness events before the process is considered not healthy"
+    integer :liveness_probe_max_interval, default: 5
+
     # The error handler must be set directly on the object.
     attr_reader :error_handler
 
@@ -210,6 +228,7 @@ module Racecar
     end
 
     def load_consumer_class(consumer_class)
+      self.consumer_class = consumer_class
       self.group_id = consumer_class.group_id || self.group_id
 
       self.group_id ||= [
@@ -226,6 +245,7 @@ module Racecar
       self.fetch_messages = consumer_class.fetch_messages || self.fetch_messages
       self.pidfile ||= "#{group_id}.pid"
     end
+    attr_accessor :consumer_class
 
     def on_error(&handler)
       @error_handler = handler
@@ -247,6 +267,39 @@ module Racecar
       producer_config
     end
 
+    def instrumenter
+      @instrumenter ||= begin
+        default_payload = { client_id: client_id, group_id: group_id }
+
+        if defined?(ActiveSupport::Notifications)
+          # ActiveSupport needs `concurrent-ruby` but doesn't `require` it.
+          require 'concurrent/utility/monotonic_time'
+          Instrumenter.new(backend: ActiveSupport::Notifications, default_payload: default_payload)
+        else
+          logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
+          NullInstrumenter
+        end
+      end
+    end
+    attr_writer :instrumenter
+
+    def install_liveness_probe
+      liveness_probe.tap(&:install)
+    end
+
+    def liveness_probe
+      require "active_support/notifications"
+      @liveness_probe ||= LivenessProbe.new(
+        ActiveSupport::Notifications,
+        liveness_probe_file_path,
+        liveness_probe_max_interval
+      )
+    end
+
+    def rebalance_listener
+      RebalanceListener.new(self)
+    end
+
     private
 
     def rdkafka_security_config

data/lib/racecar/consumer_set.rb

@@ -69,7 +69,10 @@ module Racecar
 
     def current
       @consumers[@consumer_id_iterator.peek] ||= begin
-        consumer = Rdkafka::Config.new(rdkafka_config(current_subscription)).consumer
+        consumer_config = Rdkafka::Config.new(rdkafka_config(current_subscription))
+        consumer_config.consumer_rebalance_listener = @config.rebalance_listener
+
+        consumer = consumer_config.consumer
         @instrumenter.instrument('join_group') do
           consumer.subscribe current_subscription.topic
         end
@@ -140,7 +143,7 @@ module Racecar
         @logger.debug "No time remains for polling messages. Will try on next call."
         return nil
       elsif wait_ms >= remain_ms
-        @logger.error "Only #{remain_ms}ms left, but want to wait for #{wait_ms}ms before poll. Will retry on next call."
+        @logger.warn "Only #{remain_ms}ms left, but want to wait for #{wait_ms}ms before poll. Will retry on next call."
         @previous_retries = try
         return nil
       elsif wait_ms > 0

data/lib/racecar/ctl.rb

@@ -32,6 +32,17 @@ module Racecar
       @command = command
     end
 
+    def liveness_probe(args)
+      require "racecar/liveness_probe"
+      parse_options!(args)
+
+      if ENV["RAILS_ENV"]
+        Racecar.config.load_file("config/racecar.yml", ENV["RAILS_ENV"])
+      end
+
+      Racecar.config.liveness_probe.check_liveness_within_interval!
+    end
+
     def status(args)
       parse_options!(args)
 

data/lib/racecar/daemon.rb

@@ -54,7 +54,7 @@ module Racecar
     end
 
     def pid
-      if File.exists?(pidfile)
+      if File.exist?(pidfile)
         File.read(pidfile).to_i
       else
         nil
@@ -89,7 +89,7 @@ module Racecar
       end
 
       at_exit do
-        File.delete(pidfile) if File.exists?(pidfile)
+        File.delete(pidfile) if File.exist?(pidfile)
       end
     rescue Errno::EEXIST
       check_pid

data/lib/racecar/datadog.rb

@@ -211,6 +211,10 @@ module Racecar
           topic: topic,
         }
 
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
         # This gets us the write rate.
         increment("producer.produce.messages", tags: tags.merge(topic: topic))
 
@@ -245,6 +249,67 @@ module Racecar
         increment("producer.ack.messages", tags: tags)
       end
 
+      def produce_delivery_error(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+        }
+
+        increment("producer.produce.delivery.errors", tags: tags)
+      end
+
+      def produce_async(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+        buffer_size = event.payload.fetch(:buffer_size)
+
+        tags = {
+          client: client,
+          topic: topic,
+        }
+
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
+        # This gets us the write rate.
+        increment("producer.produce.messages", tags: tags.merge(topic: topic))
+
+        # Information about typical/average/95p message size.
+        histogram("producer.produce.message_size", message_size, tags: tags.merge(topic: topic))
+
+        # Aggregate message size.
+        count("producer.produce.message_size.sum", message_size, tags: tags.merge(topic: topic))
+
+        # This gets us the avg/max buffer size per producer.
+        histogram("producer.buffer.size", buffer_size, tags: tags)
+      end
+
+      def produce_sync(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+
+        tags = {
+          client: client,
+          topic: topic,
+        }
+
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
+
+        # This gets us the write rate.
+        increment("producer.produce.messages", tags: tags.merge(topic: topic))
+
+        # Information about typical/average/95p message size.
+        histogram("producer.produce.message_size", message_size, tags: tags.merge(topic: topic))
+
+        # Aggregate message size.
+        count("producer.produce.message_size.sum", message_size, tags: tags.merge(topic: topic))
+      end
+
       attach_to "racecar"
     end
   end

data/lib/racecar/delivery_callback.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Racecar
+  class DeliveryCallback
+    attr_reader :instrumenter
+
+    def initialize(instrumenter:)
+      @instrumenter = instrumenter
+    end
+
+    def call(delivery_report)
+      if delivery_report.error.to_i.zero?
+        payload = {
+          offset: delivery_report.offset,
+          partition: delivery_report.partition
+        }
+        instrumenter.instrument("acknowledged_message", payload)
+      else
+        payload = {
+          partition: delivery_report.partition,
+          exception: delivery_report.error
+        }
+        instrumenter.instrument("produce_delivery_error", payload)
+      end
+    end
+  end
+end

data/lib/racecar/instrumenter.rb

@@ -9,16 +9,9 @@ module Racecar
     NAMESPACE = "racecar"
     attr_reader :backend
 
-    def initialize(default_payload = {})
+    def initialize(backend:, default_payload: {})
+      @backend = backend
       @default_payload = default_payload
-
-      @backend = if defined?(ActiveSupport::Notifications)
-        # ActiveSupport needs `concurrent-ruby` but doesn't `require` it.
-        require 'concurrent/utility/monotonic_time'
-        ActiveSupport::Notifications
-      else
-        NullInstrumenter
-      end
     end
 
     def instrument(event_name, payload = {}, &block)

data/lib/racecar/liveness_probe.rb

@@ -0,0 +1,78 @@
+require "fileutils"
+
+module Racecar
+  class LivenessProbe
+    def initialize(message_bus, file_path, max_interval)
+      @message_bus = message_bus
+      @file_path = file_path
+      @max_interval = max_interval
+      @subscribers = []
+    end
+
+    attr_reader :message_bus, :file_path, :max_interval, :subscribers
+    private     :message_bus, :file_path, :max_interval, :subscribers
+
+    def check_liveness_within_interval!
+      unless liveness_event_within_interval?
+        $stderr.puts "Racecar healthcheck failed: No liveness within interval #{max_interval}s. Last liveness at #{last_liveness_event_at}, #{elapsed_since_liveness_event} seconds ago."
+        Process.exit(1)
+      end
+    end
+
+    def liveness_event_within_interval?
+      elapsed_since_liveness_event < max_interval
+    rescue Errno::ENOENT
+      $stderr.puts "Racecar healthcheck failed: Liveness file not found `#{file_path}`"
+      Process.exit(1)
+    end
+
+    def install
+      unless file_path && file_writeable?
+        raise(
+          "Liveness probe configuration error: `liveness_probe_file_path` must be set to a writable file path.\n" \
+            "  Set `RACECAR_LIVENESS_PROBE_FILE_PATH` and `RACECAR_LIVENESS_MAX_INTERVAL` environment variables."
+        )
+      end
+
+      subscribers << message_bus.subscribe("start_main_loop.racecar") do
+        touch_liveness_file
+      end
+
+      subscribers = message_bus.subscribe("shut_down.racecar") do
+        delete_liveness_file
+      end
+
+      nil
+    end
+
+    def uninstall
+      subscribers.each { |s| message_bus.unsubscribe(s) }
+    end
+
+    private
+
+    def elapsed_since_liveness_event
+      Time.now - last_liveness_event_at
+    end
+
+    def last_liveness_event_at
+      File.mtime(file_path)
+    end
+
+    def touch_liveness_file
+      FileUtils.touch(file_path)
+    end
+
+    def delete_liveness_file
+      FileUtils.rm_rf(file_path)
+    end
+
+    def file_writeable?
+      File.write(file_path, "")
+      File.unlink(file_path)
+      true
+    rescue
+      false
+    end
+  end
+end

data/lib/racecar/parallel_runner.rb

@@ -37,6 +37,10 @@ module Racecar
       wait_for_exit
     end
 
+    def stop
+      terminate_workers
+    end
+
     private
 
     attr_accessor :workers

data/lib/racecar/producer.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "racecar/message_delivery_error"
+require "racecar/delivery_callback"
+
+at_exit do
+  Racecar::Producer.shutdown!
+end
+
+module Racecar
+  class Producer
+
+    @@mutex = Mutex.new
+
+    class << self
+      def shutdown!
+        @@mutex.synchronize do
+          if !@internal_producer.nil?
+            @internal_producer.close
+          end
+        end
+      end
+    end
+
+    def initialize(config: nil, logger: nil, instrumenter: NullInstrumenter)
+      @config = config
+      @logger = logger
+      @delivery_handles = []
+      @instrumenter = instrumenter
+      @batching = false
+      @internal_producer = init_internal_producer(config)
+    end
+
+    def init_internal_producer(config)
+      @@mutex.synchronize do
+        @@init_internal_producer ||= begin
+          # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+          producer_config = {
+            "bootstrap.servers"      => config.brokers.join(","),
+            "client.id"              => config.client_id,
+            "statistics.interval.ms" => config.statistics_interval_ms,
+            "message.timeout.ms"     => config.message_timeout * 1000,
+          }
+          producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
+          producer_config.merge!(config.rdkafka_producer)
+          Rdkafka::Config.new(producer_config).producer.tap do |producer|
+            producer.delivery_callback = DeliveryCallback.new(instrumenter: @instrumenter)
+          end
+        end
+      end
+    end
+
+    # fire and forget - you won't get any guarantees or feedback from
+    # Racecar on the status of the message and it won't halt execution
+    # of the rest of your code.
+    def produce_async(value:, topic:, **options)
+      with_instrumentation(action: "produce_async", value: value, topic: topic, **options) do
+        begin
+          handle = internal_producer.produce(payload: value, topic: topic, **options)
+          @delivery_handles << handle if @batching
+        rescue Rdkafka::RdkafkaError => e
+          raise MessageDeliveryError.new(e, handle)
+        end
+      end
+
+      nil
+    end
+
+    # synchronous message production - will wait until the delivery handle succeeds, fails or times out.
+    def produce_sync(value:, topic:, **options)
+      with_instrumentation(action: "produce_sync", value: value, topic: topic, **options) do
+        begin
+          handle = internal_producer.produce(payload: value, topic: topic, **options)
+          deliver_with_error_handling(handle)
+        rescue Rdkafka::RdkafkaError => e
+          raise MessageDeliveryError.new(e, handle)
+        end
+      end
+
+      nil
+    end
+
+    # Blocks until all messages that have been asynchronously produced in the block have been delivered.
+    # Usage:
+    # messages = [
+    #             {value: "message1", topic: "topic1"},
+    #             {value: "message2", topic: "topic1"},
+    #             {value: "message3", topic: "topic2"}
+    #             ]
+    # Racecar.wait_for_delivery {
+    #   messages.each do |msg|
+    #     Racecar.produce_async(value: msg[:value], topic: msg[:topic])
+    #   end
+    # }
+    def wait_for_delivery
+      @batching = true
+      @delivery_handles.clear
+      yield
+      @delivery_handles.each do |handle|
+        deliver_with_error_handling(handle)
+      end
+    ensure
+      @delivery_handles.clear
+      @batching = false
+
+      nil
+    end
+
+    private
+
+    attr_reader :internal_producer
+
+    def deliver_with_error_handling(handle)
+      handle.wait
+    rescue Rdkafka::AbstractHandle::WaitTimeoutError => e
+      partition = MessageDeliveryError.partition_from_delivery_handle(handle)
+      @logger.warn "Still trying to deliver message to (partition #{partition})... (will try up to Racecar.config.message_timeout)"
+      retry
+    rescue Rdkafka::RdkafkaError => e
+      raise MessageDeliveryError.new(e, handle)
+    end
+
+    def with_instrumentation(action:, value:, topic:, **options)
+      message_size = value.respond_to?(:bytesize) ? value.bytesize : 0
+      instrumentation_payload = {
+        value: value,
+        topic: topic,
+        message_size: message_size,
+        buffer_size: @delivery_handles.size,
+        key: options.fetch(:key, nil),
+        partition: options.fetch(:partition, nil),
+        partition_key: options.fetch(:partition_key, nil)
+      }
+      @instrumenter.instrument(action, instrumentation_payload) do
+        yield
+      end
+    end
+  end
+end

data/lib/racecar/rebalance_listener.rb

@@ -0,0 +1,22 @@
+module Racecar
+  class RebalanceListener
+    def initialize(config)
+      @config = config
+      @consumer_class = config.consumer_class
+    end
+
+    attr_reader :config, :consumer_class
+
+    def on_partitions_assigned(_consumer, topic_partition_list)
+      consumer_class.respond_to?(:on_partitions_assigned) &&
+        consumer_class.on_partitions_assigned(topic_partition_list.to_h)
+    rescue
+    end
+
+    def on_partitions_revoked(_consumer, topic_partition_list)
+      consumer_class.respond_to?(:on_partitions_revoked) &&
+        consumer_class.on_partitions_revoked(topic_partition_list.to_h)
+    rescue
+    end
+  end
+end

data/lib/racecar/runner.rb

@@ -5,6 +5,7 @@ require "racecar/pause"
 require "racecar/message"
 require "racecar/message_delivery_error"
 require "racecar/erroneous_state_error"
+require "racecar/delivery_callback"
 
 module Racecar
   class Runner
@@ -67,6 +68,8 @@ module Racecar
       loop do
         break if @stop_requested
         resume_paused_partitions
+
+        @instrumenter.instrument("start_main_loop", instrumentation_payload)
         @instrumenter.instrument("main_loop", instrumentation_payload) do
           case process_method
           when :batch then
@@ -94,6 +97,7 @@ module Racecar
     ensure
       producer.close
       Racecar::Datadog.close if Object.const_defined?("Racecar::Datadog")
+      @instrumenter.instrument("shut_down", instrumentation_payload || {})
     end
 
     def stop
@@ -138,7 +142,7 @@ module Racecar
 
     def producer
       @producer ||= Rdkafka::Config.new(producer_config).producer.tap do |producer|
-        producer.delivery_callback = delivery_callback
+        producer.delivery_callback = Racecar::DeliveryCallback.new(instrumenter: @instrumenter)
       end
     end
 
@@ -149,22 +153,14 @@ module Racecar
         "client.id"              => config.client_id,
         "statistics.interval.ms" => config.statistics_interval_ms,
         "message.timeout.ms"     => config.message_timeout * 1000,
+        "partitioner"            => config.partitioner.to_s,
       }
+
       producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
       producer_config.merge!(config.rdkafka_producer)
       producer_config
     end
 
-    def delivery_callback
-      ->(delivery_report) do
-        payload = {
-          offset: delivery_report.offset,
-          partition: delivery_report.partition
-        }
-        @instrumenter.instrument("acknowledged_message", payload)
-      end
-    end
-
     def install_signal_handlers
       # Stop the consumer on SIGINT, SIGQUIT or SIGTERM.
       trap("QUIT") { stop }

data/lib/racecar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Racecar
-  VERSION = "2.8.2"
+  VERSION = "2.9.0"
 end

data/lib/racecar.rb

@@ -8,6 +8,7 @@ require "racecar/consumer"
 require "racecar/consumer_set"
 require "racecar/runner"
 require "racecar/parallel_runner"
+require "racecar/producer"
 require "racecar/config"
 require "racecar/version"
 require "ensure_hash_compact"
@@ -39,25 +40,42 @@ module Racecar
     config.logger = logger
   end
 
-  def self.instrumenter
-    @instrumenter ||= begin
-      default_payload = { client_id: config.client_id, group_id: config.group_id }
+  def self.produce_async(value:, topic:, **options)
+    producer.produce_async(value: value, topic: topic, **options)
+  end
 
-      Instrumenter.new(default_payload).tap do |instrumenter|
-        if instrumenter.backend == NullInstrumenter
-          logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
-        end
+  def self.produce_sync(value:, topic:, **options)
+    producer.produce_sync(value: value, topic: topic, **options)
+  end  
+  
+  def self.wait_for_delivery(&block)
+    producer.wait_for_delivery(&block)
+  end
+
+  def self.producer
+    Thread.current[:racecar_producer] ||= begin
+      if config.datadog_enabled
+        require "racecar/datadog"
       end
+      Racecar::Producer.new(config: config, logger: logger, instrumenter: instrumenter)
     end
   end
 
+  def self.instrumenter
+    config.instrumenter
+  end
+
   def self.run(processor)
-    runner = Runner.new(processor, config: config, logger: logger, instrumenter: instrumenter)
+    runner(processor).run
+  end
+
+  def self.runner(processor)
+    runner = Runner.new(processor, config: config, logger: logger, instrumenter: config.instrumenter)
 
     if config.parallel_workers && config.parallel_workers > 1
-      ParallelRunner.new(runner: runner, config: config, logger: logger).run
+      ParallelRunner.new(runner: runner, config: config, logger: logger)
     else
-      runner.run
+      runner
     end
   end
 end

data/racecar.gemspec

@@ -26,10 +26,10 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "rdkafka",   "~> 0.12.0"
 
   spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
-  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "rake", "> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 6.0.0"
-  spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
+  spec.add_development_dependency "activesupport"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.8.2
+  version: 2.9.0
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-07-12 00:00:00.000000000 Z
+date: 2023-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -60,7 +60,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -141,20 +141,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '0'
 description: 
 email:
 - dschierbeck@zendesk.com
@@ -165,9 +159,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
 - ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
+- ".github/workflows/publish.yml"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md
@@ -200,15 +194,19 @@ files:
 - lib/racecar/ctl.rb
 - lib/racecar/daemon.rb
 - lib/racecar/datadog.rb
+- lib/racecar/delivery_callback.rb
 - lib/racecar/erroneous_state_error.rb
 - lib/racecar/heroku.rb
 - lib/racecar/instrumenter.rb
+- lib/racecar/liveness_probe.rb
 - lib/racecar/message.rb
 - lib/racecar/message_delivery_error.rb
 - lib/racecar/null_instrumenter.rb
 - lib/racecar/parallel_runner.rb
 - lib/racecar/pause.rb
+- lib/racecar/producer.rb
 - lib/racecar/rails_config_file_loader.rb
+- lib/racecar/rebalance_listener.rb
 - lib/racecar/runner.rb
 - lib/racecar/version.rb
 - racecar.gemspec
@@ -231,7 +229,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: A framework for running Kafka consumers

data/.circleci/config.yml

@@ -1,56 +0,0 @@
-version: 2.1
-orbs:
-  ruby: circleci/ruby@0.1.2
-
-jobs:
-  build:
-    docker:
-      - image: circleci/ruby:2.6.3-stretch-node
-    executor: ruby/default
-    steps:
-      - checkout
-      - run:
-          name: Which bundler?
-          command: bundle -v
-      - ruby/bundle-install
-      - run: bundle exec rspec --exclude-pattern='spec/integration/*_spec.rb'
-  integration-tests:
-    docker:
-      - image: circleci/ruby:2.6.3-stretch-node
-      - image: wurstmeister/zookeeper
-      - image: wurstmeister/kafka:2.11-2.0.0
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9092
-          KAFKA_PORT: 9092
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:2.11-2.0.0
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9093
-          KAFKA_PORT: 9093
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:2.11-2.0.0
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9094
-          KAFKA_PORT: 9094
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-    executor: ruby/default
-    steps:
-      - checkout
-      - run:
-          name: Which bundler?
-          command: bundle -v
-      - ruby/bundle-install
-      - run: bundle exec rspec --pattern='spec/integration/*_spec.rb'
-
-workflows:
-  version: 2
-  test:
-    jobs:
-      - build
-      - integration-tests