racecar 2.8.2 → 2.9.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4cbf6ca4837bdd3893c8342936f9d5ea115a5b3a585e0c843e62171e00038e72
-  data.tar.gz: d8603c46cf801d0f41f017d5b27b35e0d2b195eec5be11a1f7b6bf77413726b4
+  metadata.gz: 364239e804c99b816c0a37802b605ff4035d2d4c51abc0b161e11609268349ea
+  data.tar.gz: 4d7bd65a04b8c914640a2edc11f3c7521b2e061db8eb3c215ba29a785f031e12
 SHA512:
-  metadata.gz: 78bdedfe7bd888a3716f5a29f28810f4c4d2f03e43c33b8e47ed08607155211fca1d47f4eab6a4f0def6976bb7ca3263117e52d2e0fa7f0a721c56ec70c7ee3d
-  data.tar.gz: 2bf6e2b50980add68363b59718f54403caada58abcb404a9018ac4304519bbbb7ed920578d37bcadefad63964812e7374d367298f403577725eef79af269c604
+  metadata.gz: c60037cd0dd5477a77a5fddd42003a3d4be67c38fb1c812cfff698cdb8fb7cd62e7e65097b6508c2a19a768daad5139f3eb469e91f0ef3ff3fe9535f9fd86a8a
+  data.tar.gz: 88d3ffcfa27b6bcb89b454151db353165ab62fa0dd42586c177d05ea6091663227b8d78f14180e69c4937417d0f761fc5666657be1320d44d54eeebcc5d78601

data/.github/workflows/publish.yml

@@ -0,0 +1,12 @@
+name: Publish Gem
+
+on:
+  push:
+    tags: v*
+
+jobs:
+  call-workflow:
+    uses: zendesk/gw/.github/workflows/ruby-gem-publication.yml@main
+    secrets:
+      RUBY_GEMS_API_KEY: ${{ secrets.RUBY_GEMS_API_KEY }}
+      RUBY_GEMS_TOTP_DEVICE: ${{ secrets.RUBY_GEMS_TOTP_DEVICE }}

data/CHANGELOG.md

@@ -1,15 +1,20 @@
 # Changelog
 
-## Unreleased
+## v2.9.0.beta1
+
+* Add `partitioner` producer config option to allow changing the strategy to
+  determine which topic partition a message is written to when racecar
+  produces a kafka message
+* Add built-in liveness probe for Kubernetes deployments.
 
 ## v2.8.2
-* Handles ErroneousStateError, in previous versions the consumer would do several unecessary group leave/joins. The log level is also changed to WARN instead of ERROR. ([#285](https://github.com/zendesk/racecar/pull/295))
+* Handles ErroneousStateError, in previous versions the consumer would do several unecessary group leave/joins. The log level is also changed to WARN instead of ERROR. ([#295](https://github.com/zendesk/racecar/pull/295))
 
 ## v2.8.1
 * Adds new ErroneousStateError to racecar in order to give more information on this new possible exception.
 
 ## v2.8.0
-* Update librdkafka version from 1.8.2 to 1.9.0 by upgrading from rdkafka 0.10.0 to 0.12.0. ([#283](https://github.com/zendesk/racecar/pull/293))
+* Update librdkafka version from 1.8.2 to 1.9.0 by upgrading from rdkafka 0.10.0 to 0.12.0. ([#293](https://github.com/zendesk/racecar/pull/293))
 
 ## v2.7.0
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    racecar (2.8.2.beta)
+    racecar (2.9.0.beta1)
       king_konf (~> 1.0.0)
       rdkafka (~> 0.12.0)
 
@@ -20,9 +20,9 @@ GEM
     ffi (1.15.5)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    king_konf (1.0.0)
+    king_konf (1.0.1)
     method_source (1.0.0)
-    mini_portile2 (2.8.0)
+    mini_portile2 (2.8.1)
     minitest (5.14.4)
     pry (0.13.1)
       coderay (~> 1.1)
@@ -47,7 +47,7 @@ GEM
     rspec-support (3.10.2)
     thread_safe (0.3.6)
     timecop (0.9.2)
-    tzinfo (1.2.9)
+    tzinfo (1.2.10)
       thread_safe (~> 0.1)
 
 PLATFORMS

data/README.md

@@ -20,6 +20,7 @@ The framework is based on [rdkafka-ruby](https://github.com/appsignal/rdkafka-ru
    8. [Logging](#logging)
    9. [Operations](#operations)
    10. [Upgrading from v1 to v2](#upgrading-from-v1-to-v2)
+   11. [Compression](#compression)
 3. [Development](#development)
 4. [Contributing](#contributing)
 5. [Support and Discussion](#support-and-discussion)
@@ -245,6 +246,69 @@ The `deliver!` method can be used to block until the broker received all queued
 
 You can set message headers by passing a `headers:` option with a Hash of headers.
 
+### Standalone Producer
+
+Racecar provides a standalone producer to publish messages to Kafka directly from your Rails application:
+```
+# app/controllers/comments_controller.rb
+class CommentsController < ApplicationController
+  def create
+    @comment = Comment.create!(params)
+
+    # This will publish a JSON representation of the comment to the `comments` topic
+    # in Kafka. Make sure to create the topic first, or this may fail.
+    Racecar.produce_sync(value:comment.to_json, topic: "comments")
+  end
+end
+```
+
+The above example will block the server process until the message has been delivered. If you want deliveries to happen in the background in order to free up your server processes more quickly, call #deliver_async instead:
+```
+# app/controllers/comments_controller.rb
+class CommentsController < ApplicationController
+  def show
+    @comment = Comment.find(params[:id])
+
+    event = {
+      name: "comment_viewed",
+      data: {
+        comment_id: @comment.id,
+        user_id: current_user.id
+      }
+    }
+
+    # By delivering messages asynchronously you free up your server processes faster.
+    Racecar.produce_async(value: event.to_json, topic: "activity")
+  end
+end
+```
+In addition to improving response time, delivering messages asynchronously also protects your application against Kafka availability issues -- if messages cannot be delivered, they'll be buffered for later and retried automatically.
+
+A third method is to produce messages first (without delivering the messages to Kafka yet), and deliver them synchronously later.
+
+```
+ # app/controllers/comments_controller.rb
+ class CommentsController < ApplicationController
+   def create
+     @comment = Comment.create!(params)
+
+     event = {
+       name: "comment_created",
+       data: {
+         comment_id: @comment.id
+         user_id: current_user.id
+       }
+     }
+
+     # This will queue the two messages in the internal buffer and block server process until they are delivered.
+    Racecar.wait_for_delivery do
+      Racecar.produce_async(comment.to_json, topic: "comments")
+      Racecar.produce_async(event.to_json, topic: "activity")
+    end
+   end
+ end
+ ```
+
 ### Configuration
 
 Racecar provides a flexible way to configure your consumer in a way that feels at home in a Rails application. If you haven't already, run `bundle exec rails generate racecar:install` in order to generate a config file. You'll get a separate section for each Rails environment, with the common configuration values in a shared `common` section.
@@ -338,6 +402,7 @@ Racecar has support for using SASL to authenticate clients using either the GSSA
 
 These settings are related to consumers that _produce messages to Kafka_.
 
+- `partitioner` – The strategy used to determine which topic partition a message is written to when Racecar produces a value to Kafka. The codec needs to be one of `consistent`, `consistent_random` `murmur2` `murmur2_random` `fnv1a` `fnv1a_random` either as a Symbol or a String, defaults to `consistent_random`
 - `producer_compression_codec` – If defined, Racecar will compress messages before writing them to Kafka. The codec needs to be one of `gzip`, `lz4`, or `snappy`, either as a Symbol or a String.
 
 #### Datadog monitoring
@@ -443,6 +508,64 @@ The important part is the `strategy.type` value, which tells Kubernetes how to u
 
 Instead, the `Recreate` update strategy should be used. It completely tears down the existing containers before starting all of the new containers simultaneously, allowing for a single synchronization stage and a much faster, more stable deployment update.
 
+#### Liveness Probe
+
+Racecar comes with a built-in liveness probe, primarily for use with Kubernetes, but useful for any deployment environment where you can periodically run a process to check the health of your consumer.
+
+To use this feature:
+- set the `liveness_probe_enabled` config option to true.
+- configure your Kubernetes deployment to run `$ racecarctl liveness_probe`
+
+
+When enabled (see config) Racecar will touch the file at `liveness_probe_file_path` each time it finishes polling Kafka and processing the messages in the batch (if any).
+
+The modified time of this file can be observed to determine when the consumer last exhibited 'liveness'.
+
+Running `racecarctl liveness_probe` will return a successful exit status if the last 'liveness' event happened within an acceptable time, `liveness_probe_max_interval`.
+
+`liveness_probe_max_interval` should be long enough to account for both the Kafka polling time of `max_wait_time` and the processing time of a full message batch.
+
+On receiving `SIGTERM`, Racecar will gracefully shut down and delete this file, causing the probe to fail immediately after exit.
+
+You may wish to tolerate more than one failed probe run to accommodate for environmental variance and clock changes.
+
+See the [Configuration section](https://github.com/zendesk/racecar#configuration) for the various ways the liveness probe can be configured, environment variables being one option.
+
+Here is an example Kubernetes liveness probe configuration:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+spec:
+  template:
+    spec:
+      containers:
+      - name: consumer
+
+        args:
+        - racecar
+        - SomeConsumer
+
+        env:
+        - name: RACECAR_LIVENESS_PROBE_ENABLED
+          value: "true"
+
+        livenessProbe:
+          exec:
+            command:
+            - racecarctl
+            - liveness_probe
+
+          # Allow up to 10 consecutive failures before terminating Pod:
+          failureThreshold: 10
+
+          # Wait 30 seconds before starting the probes:
+          initialDelaySeconds: 30
+
+          # Perform the check every 10 seconds:
+          periodSeconds: 10
+```
+
 #### Deploying to Heroku
 
 If you run your applications in Heroku and/or use the Heroku Kafka add-on, you application will be provided with 4 ENV variables that allow connecting to the cluster: `KAFKA_URL`, `KAFKA_TRUSTED_CERT`, `KAFKA_CLIENT_CERT`, and `KAFKA_CLIENT_CERT_KEY`.
@@ -479,7 +602,7 @@ Again, the recommended approach is to manage the processes using process manager
 
 ### Handling errors
 
-When processing messages from a Kafka topic, your code may encounter an error and raise an exception. The cause is typically one of two things:
+#### When processing messages from a Kafka topic, your code may encounter an error and raise an exception. The cause is typically one of two things:
 
 1. The message being processed is somehow malformed or doesn't conform with the assumptions made by the processing code.
 2. You're using some external resource such as a database or a network API that is temporarily unavailable.
@@ -514,6 +637,16 @@ end
 
 It is highly recommended that you set up an error handler. Please note that the `info` object contains different keys and values depending on whether you are using `process` or `process_batch`. See the `instrumentation_payload` object in the `process` and `process_batch` methods in the `Runner` class for the complete list.
 
+#### Errors related to Compression
+
+A sample error might look like this:
+
+```
+E, [2022-10-09T11:28:29.976548 #15] ERROR -- : (try 5/10): Error for topic subscription #<struct Racecar::Consumer::Subscription topic="support.entity_incremental.views.view_ticket_ids", start_from_beginning=false, max_bytes_per_partition=104857, additional_config={}>: Local: Not implemented (not_implemented)
+```
+
+Please see [Compression](#compression)
+
 ### Logging
 
 By default, Racecar will log to `STDOUT`. If you're using Rails, your application code will use whatever logger you've configured there.
@@ -528,7 +661,15 @@ In order to introspect the configuration of a consumer process, send it the `SIG
 
 ### Upgrading from v1 to v2
 
-In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency. In general, you should avoid rolling deploys for consumers groups, so it is likely the case that this will just work for you, but it's a good idea to check first.
+In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency. In general, you should avoid rolling deploys for consumers groups, so it is likely the case that this will just work for you, but it's a good idea to check first. 
+
+### Compression
+
+Racecar v2 requires a C library (zlib) to compress the messages before producing to the topic. If not already installed on you consumer docker container, please install using following command in Dockerfile of consumer
+
+``` 
+apt-get update && apt-get install -y libzstd-dev
+```
 
 ## Development
 

data/Rakefile

@@ -3,6 +3,9 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 
+# Pushing to rubygems is handled by a github workflow
+ENV["gem_push"] = "false"
+
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec

data/examples/batch_consumer.rb

@@ -3,8 +3,8 @@
 class BatchConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 
-  def process_batch(batch)
-    batch.messages.each do |message|
+  def process_batch(messages)
+    messages.each do |message|
       puts message.value
     end
   end

data/lib/racecar/cli.rb

@@ -5,6 +5,7 @@ require "logger"
 require "fileutils"
 require "racecar/rails_config_file_loader"
 require "racecar/daemon"
+require "racecar/liveness_probe"
 
 module Racecar
   class Cli
@@ -58,6 +59,11 @@ module Racecar
         $stderr.puts "=> Ctrl-C to shutdown consumer"
       end
 
+      if config.liveness_probe_enabled
+        $stderr.puts "=> Liveness probe enabled"
+        config.install_liveness_probe
+      end
+
       processor = consumer_class.new
       Racecar.run(processor)
       nil

data/lib/racecar/config.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
 
+require "tmpdir"
+
 require "king_konf"
 
+require "racecar/liveness_probe"
+require "racecar/instrumenter"
+
 module Racecar
   class Config < KingKonf::Config
     env_prefix :racecar
@@ -77,6 +82,9 @@ module Racecar
     desc "The log level for the Racecar logs"
     string :log_level, default: "info"
 
+    desc "The strategy used to determine which topic partition a message is written to when Racecar produces a value to Kafka; defaults to `consistent_random`"
+    symbol :partitioner, allowed_values: %i{consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random}, default: :consistent_random
+
     desc "Protocol used to communicate with brokers"
     symbol :security_protocol, allowed_values: %i{plaintext ssl sasl_plaintext sasl_ssl}
 
@@ -164,6 +172,15 @@ module Racecar
           for backward compatibility, however this can be quite memory intensive"
     integer :statistics_interval, default: 1
 
+    desc "Whether to enable liveness probe behavior (touch the file)"
+    boolean :liveness_probe_enabled, default: false
+
+    desc "Path to a file Racecar will touch to show liveness"
+    string :liveness_probe_file_path, default: "#{Dir.tmpdir}/racecar-liveness"
+
+    desc "Used only by the liveness probe: Max time (in seconds) between liveness events before the process is considered not healthy"
+    integer :liveness_probe_max_interval, default: 5
+
     # The error handler must be set directly on the object.
     attr_reader :error_handler
 
@@ -247,6 +264,35 @@ module Racecar
       producer_config
     end
 
+    def instrumenter
+      @instrumenter ||= begin
+        default_payload = { client_id: client_id, group_id: group_id }
+
+        if defined?(ActiveSupport::Notifications)
+          # ActiveSupport needs `concurrent-ruby` but doesn't `require` it.
+          require 'concurrent/utility/monotonic_time'
+          Instrumenter.new(backend: ActiveSupport::Notifications, default_payload: default_payload)
+        else
+          logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
+          NullInstrumenter
+        end
+      end
+    end
+    attr_writer :instrumenter
+
+    def install_liveness_probe
+      liveness_probe.tap(&:install)
+    end
+
+    def liveness_probe
+      require "active_support/notifications"
+      @liveness_probe ||= LivenessProbe.new(
+        ActiveSupport::Notifications,
+        liveness_probe_file_path,
+        liveness_probe_max_interval
+      )
+    end
+
     private
 
     def rdkafka_security_config

data/lib/racecar/consumer_set.rb

@@ -140,7 +140,7 @@ module Racecar
         @logger.debug "No time remains for polling messages. Will try on next call."
         return nil
       elsif wait_ms >= remain_ms
-        @logger.error "Only #{remain_ms}ms left, but want to wait for #{wait_ms}ms before poll. Will retry on next call."
+        @logger.warn "Only #{remain_ms}ms left, but want to wait for #{wait_ms}ms before poll. Will retry on next call."
         @previous_retries = try
         return nil
       elsif wait_ms > 0

data/lib/racecar/ctl.rb

@@ -32,6 +32,17 @@ module Racecar
       @command = command
     end
 
+    def liveness_probe(args)
+      require "racecar/liveness_probe"
+      parse_options!(args)
+
+      if ENV["RAILS_ENV"]
+        Racecar.config.load_file("config/racecar.yml", ENV["RAILS_ENV"])
+      end
+
+      Racecar.config.liveness_probe.check_liveness_within_interval!
+    end
+
     def status(args)
       parse_options!(args)
 

data/lib/racecar/datadog.rb

@@ -244,6 +244,50 @@ module Racecar
         # Number of messages ACK'd for the topic.
         increment("producer.ack.messages", tags: tags)
       end
+      
+      def produce_async(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+        buffer_size = event.payload.fetch(:buffer_size)
+
+        tags = {
+          client: client,
+          topic: topic,
+        }
+
+        # This gets us the write rate.
+        increment("producer.produce.messages", tags: tags.merge(topic: topic))
+
+        # Information about typical/average/95p message size.
+        histogram("producer.produce.message_size", message_size, tags: tags.merge(topic: topic))
+
+        # Aggregate message size.
+        count("producer.produce.message_size.sum", message_size, tags: tags.merge(topic: topic))
+
+        # This gets us the avg/max buffer size per producer.
+        histogram("producer.buffer.size", buffer_size, tags: tags)
+      end
+
+      def produce_sync(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+
+        tags = {
+          client: client,
+          topic: topic,
+        }
+
+        # This gets us the write rate.
+        increment("producer.produce.messages", tags: tags.merge(topic: topic))
+
+        # Information about typical/average/95p message size.
+        histogram("producer.produce.message_size", message_size, tags: tags.merge(topic: topic))
+
+        # Aggregate message size.
+        count("producer.produce.message_size.sum", message_size, tags: tags.merge(topic: topic))
+      end
 
       attach_to "racecar"
     end

data/lib/racecar/instrumenter.rb

@@ -9,16 +9,9 @@ module Racecar
     NAMESPACE = "racecar"
     attr_reader :backend
 
-    def initialize(default_payload = {})
+    def initialize(backend:, default_payload: {})
+      @backend = backend
       @default_payload = default_payload
-
-      @backend = if defined?(ActiveSupport::Notifications)
-        # ActiveSupport needs `concurrent-ruby` but doesn't `require` it.
-        require 'concurrent/utility/monotonic_time'
-        ActiveSupport::Notifications
-      else
-        NullInstrumenter
-      end
     end
 
     def instrument(event_name, payload = {}, &block)

data/lib/racecar/liveness_probe.rb

@@ -0,0 +1,78 @@
+require "fileutils"
+
+module Racecar
+  class LivenessProbe
+    def initialize(message_bus, file_path, max_interval)
+      @message_bus = message_bus
+      @file_path = file_path
+      @max_interval = max_interval
+      @subscribers = []
+    end
+
+    attr_reader :message_bus, :file_path, :max_interval, :subscribers
+    private     :message_bus, :file_path, :max_interval, :subscribers
+
+    def check_liveness_within_interval!
+      unless liveness_event_within_interval?
+        $stderr.puts "Racecar healthcheck failed: No liveness within interval #{max_interval}s. Last liveness at #{last_liveness_event_at}, #{elapsed_since_liveness_event} seconds ago."
+        Process.exit(1)
+      end
+    end
+
+    def liveness_event_within_interval?
+      elapsed_since_liveness_event < max_interval
+    rescue Errno::ENOENT
+      $stderr.puts "Racecar healthcheck failed: Liveness file not found `#{file_path}`"
+      Process.exit(1)
+    end
+
+    def install
+      unless file_path && file_writeable?
+        raise(
+          "Liveness probe configuration error: `liveness_probe_file_path` must be set to a writable file path.\n" \
+            "  Set `RACECAR_LIVENESS_PROBE_FILE_PATH` and `RACECAR_LIVENESS_MAX_INTERVAL` environment variables."
+        )
+      end
+
+      subscribers << message_bus.subscribe("start_main_loop.racecar") do
+        touch_liveness_file
+      end
+
+      subscribers = message_bus.subscribe("shut_down.racecar") do
+        delete_liveness_file
+      end
+
+      nil
+    end
+
+    def uninstall
+      subscribers.each { |s| message_bus.unsubscribe(s) }
+    end
+
+    private
+
+    def elapsed_since_liveness_event
+      Time.now - last_liveness_event_at
+    end
+
+    def last_liveness_event_at
+      File.mtime(file_path)
+    end
+
+    def touch_liveness_file
+      FileUtils.touch(file_path)
+    end
+
+    def delete_liveness_file
+      FileUtils.rm_rf(file_path)
+    end
+
+    def file_writeable?
+      File.write(file_path, "")
+      File.unlink(file_path)
+      true
+    rescue
+      false
+    end
+  end
+end

data/lib/racecar/producer.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "racecar/message_delivery_error"
+
+at_exit do
+  Racecar::Producer.shutdown!
+end
+
+module Racecar
+  class Producer
+
+    @@mutex = Mutex.new
+
+    class << self
+      def shutdown!
+        @@mutex.synchronize do
+          if !@internal_producer.nil?
+            @internal_producer.close
+          end
+        end
+      end
+    end
+
+    def initialize(config: nil, logger: nil, instrumenter: NullInstrumenter)
+      @config = config
+      @logger = logger
+      @delivery_handles = []
+      @instrumenter = instrumenter
+      @batching = false
+      @internal_producer = init_internal_producer(config)
+    end
+
+    def init_internal_producer(config)
+      @@mutex.synchronize do
+        @@init_internal_producer ||= begin
+          # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+          producer_config = {
+            "bootstrap.servers"      => config.brokers.join(","),
+            "client.id"              => config.client_id,
+            "statistics.interval.ms" => config.statistics_interval_ms,
+            "message.timeout.ms"     => config.message_timeout * 1000,
+          }
+          producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
+          producer_config.merge!(config.rdkafka_producer)
+          Rdkafka::Config.new(producer_config).producer
+        end
+      end
+    end
+
+
+    # fire and forget - you won't get any guarantees or feedback from 
+    # Racecar on the status of the message and it won't halt execution
+    # of the rest of your code.
+    def produce_async(value:, topic:, **options)
+      with_instrumentation(action: "produce_async", value: value, topic: topic, **options) do
+        handle = internal_producer.produce(payload: value, topic: topic, **options)
+        @delivery_handles << handle if @batching
+      end
+
+      nil
+    end
+
+    # synchronous message production - will wait until the delivery handle succeeds, fails or times out.
+    def produce_sync(value:, topic:, **options)
+      with_instrumentation(action: "produce_sync", value: value, topic: topic, **options) do
+        handle = internal_producer.produce(payload: value, topic: topic, **options)
+        deliver_with_error_handling(handle)
+      end
+
+      nil
+    end
+
+    # Blocks until all messages that have been asynchronously produced in the block have been delivered.
+    # Usage:
+    # messages = [
+    #             {value: "message1", topic: "topic1"},
+    #             {value: "message2", topic: "topic1"},
+    #             {value: "message3", topic: "topic2"}
+    #             ]
+    # Racecar.wait_for_delivery {
+    #   messages.each do |msg|
+    #     Racecar.produce_async(value: msg[:value], topic: msg[:topic])
+    #   end
+    # }
+    def wait_for_delivery
+      @batching = true
+      @delivery_handles.clear
+      yield
+      @delivery_handles.each do |handle|
+        deliver_with_error_handling(handle)
+      end
+    ensure
+      @delivery_handles.clear
+      @batching = false
+
+      nil
+    end
+
+    private
+
+    attr_reader :internal_producer
+
+    def deliver_with_error_handling(handle)
+      handle.wait
+    rescue Rdkafka::AbstractHandle::WaitTimeoutError => e
+      partition = MessageDeliveryError.partition_from_delivery_handle(handle)
+      @logger.warn "Still trying to deliver message to (partition #{partition})... (will try up to Racecar.config.message_timeout)"
+      retry
+    rescue Rdkafka::RdkafkaError => e
+      raise MessageDeliveryError.new(e, handle)
+    end
+
+    def with_instrumentation(action:, value:, topic:, **options)
+      message_size = value.respond_to?(:bytesize) ? value.bytesize : 0
+      instrumentation_payload = {
+        value: value,
+        topic: topic,
+        message_size: message_size,
+        buffer_size: @delivery_handles.size,
+        key: options.fetch(:key, nil),
+        partition: options.fetch(:partition, nil),
+        partition_key: options.fetch(:partition_key, nil)
+      }
+      @instrumenter.instrument(action, instrumentation_payload) do
+        yield
+      end
+    end
+  end
+end

data/lib/racecar/runner.rb

@@ -67,6 +67,8 @@ module Racecar
       loop do
         break if @stop_requested
         resume_paused_partitions
+
+        @instrumenter.instrument("start_main_loop", instrumentation_payload)
         @instrumenter.instrument("main_loop", instrumentation_payload) do
           case process_method
           when :batch then
@@ -94,6 +96,7 @@ module Racecar
     ensure
       producer.close
       Racecar::Datadog.close if Object.const_defined?("Racecar::Datadog")
+      @instrumenter.instrument("shut_down", instrumentation_payload || {})
     end
 
     def stop
@@ -149,7 +152,9 @@ module Racecar
         "client.id"              => config.client_id,
         "statistics.interval.ms" => config.statistics_interval_ms,
         "message.timeout.ms"     => config.message_timeout * 1000,
+        "partitioner"            => config.partitioner.to_s,
       }
+
       producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
       producer_config.merge!(config.rdkafka_producer)
       producer_config

data/lib/racecar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Racecar
-  VERSION = "2.8.2"
+  VERSION = "2.9.0.beta1"
 end

data/lib/racecar.rb

@@ -8,6 +8,7 @@ require "racecar/consumer"
 require "racecar/consumer_set"
 require "racecar/runner"
 require "racecar/parallel_runner"
+require "racecar/producer"
 require "racecar/config"
 require "racecar/version"
 require "ensure_hash_compact"
@@ -39,20 +40,33 @@ module Racecar
     config.logger = logger
   end
 
-  def self.instrumenter
-    @instrumenter ||= begin
-      default_payload = { client_id: config.client_id, group_id: config.group_id }
+  def self.produce_async(value:, topic:, **options)
+    producer.produce_async(value: value, topic: topic, **options)
+  end
+
+  def self.produce_sync(value:, topic:, **options)
+    producer.produce_sync(value: value, topic: topic, **options)
+  end  
+  
+  def self.wait_for_delivery(&block)
+    producer.wait_for_delivery(&block)
+  end
 
-      Instrumenter.new(default_payload).tap do |instrumenter|
-        if instrumenter.backend == NullInstrumenter
-          logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
-        end
+  def self.producer
+    Thread.current[:racecar_producer] ||= begin
+      if config.datadog_enabled
+        require "racecar/datadog"
       end
+      Racecar::Producer.new(config: config, logger: logger, instrumenter: instrumenter)
     end
   end
 
+  def self.instrumenter
+    config.instrumenter
+  end
+
   def self.run(processor)
-    runner = Runner.new(processor, config: config, logger: logger, instrumenter: instrumenter)
+    runner = Runner.new(processor, config: config, logger: logger, instrumenter: config.instrumenter)
 
     if config.parallel_workers && config.parallel_workers > 1
       ParallelRunner.new(runner: runner, config: config, logger: logger).run

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.8.2
+  version: 2.9.0.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-07-12 00:00:00.000000000 Z
+date: 2023-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -165,9 +165,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
 - ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
+- ".github/workflows/publish.yml"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md
@@ -203,11 +203,13 @@ files:
 - lib/racecar/erroneous_state_error.rb
 - lib/racecar/heroku.rb
 - lib/racecar/instrumenter.rb
+- lib/racecar/liveness_probe.rb
 - lib/racecar/message.rb
 - lib/racecar/message_delivery_error.rb
 - lib/racecar/null_instrumenter.rb
 - lib/racecar/parallel_runner.rb
 - lib/racecar/pause.rb
+- lib/racecar/producer.rb
 - lib/racecar/rails_config_file_loader.rb
 - lib/racecar/runner.rb
 - lib/racecar/version.rb
@@ -227,11 +229,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: A framework for running Kafka consumers

data/.circleci/config.yml

@@ -1,56 +0,0 @@
-version: 2.1
-orbs:
-  ruby: circleci/ruby@0.1.2
-
-jobs:
-  build:
-    docker:
-      - image: circleci/ruby:2.6.3-stretch-node
-    executor: ruby/default
-    steps:
-      - checkout
-      - run:
-          name: Which bundler?
-          command: bundle -v
-      - ruby/bundle-install
-      - run: bundle exec rspec --exclude-pattern='spec/integration/*_spec.rb'
-  integration-tests:
-    docker:
-      - image: circleci/ruby:2.6.3-stretch-node
-      - image: wurstmeister/zookeeper
-      - image: wurstmeister/kafka:2.11-2.0.0
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9092
-          KAFKA_PORT: 9092
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:2.11-2.0.0
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9093
-          KAFKA_PORT: 9093
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-      - image: wurstmeister/kafka:2.11-2.0.0
-        environment:
-          KAFKA_ADVERTISED_HOST_NAME: localhost
-          KAFKA_ADVERTISED_PORT: 9094
-          KAFKA_PORT: 9094
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_DELETE_TOPIC_ENABLE: true
-    executor: ruby/default
-    steps:
-      - checkout
-      - run:
-          name: Which bundler?
-          command: bundle -v
-      - ruby/bundle-install
-      - run: bundle exec rspec --pattern='spec/integration/*_spec.rb'
-
-workflows:
-  version: 2
-  test:
-    jobs:
-      - build
-      - integration-tests