racecar 2.0.0 → 2.10.0.beta2

data/examples/batch_consumer.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 class BatchConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 
-  def process_batch(batch)
-    batch.messages.each do |message|
+  def process_batch(messages)
+    messages.each do |message|
       puts message.value
     end
   end

data/examples/cat_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class CatConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/examples/producing_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ProducingConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/exe/racecar

@@ -3,19 +3,42 @@
 require "racecar"
 require "racecar/cli"
 
-begin
-  Racecar::Cli.main(ARGV)
-rescue SignalException => e
-  # We might receive SIGTERM before our signal handler is installed.
-  if Signal.signame(e.signo) == "TERM"
-    exit(0)
-  else
-    raise
+module Racecar
+  class << self
+    def start(argv)
+      Cli.main(argv)
+    rescue SignalException => e
+      # We might receive SIGTERM before our signal handler is installed.
+      if Signal.signame(e.signo) == "TERM"
+        exit(0)
+      else
+        raise
+      end
+    rescue SystemExit
+      raise
+    rescue Exception => e
+      $stderr.puts "=> Crashed: #{exception_with_causes(e)}\n#{e.backtrace.join("\n")}"
+
+      Racecar.config.error_handler.call(e)
+
+      exit(1)
+    else
+      exit(0)
+    end
+
+    private
+
+    def exception_with_causes(e)
+      result = +"#{e.class}: #{e}"
+      if e.cause
+        result << "\n"
+        result << "--- Caused by: ---\n"
+        result << exception_with_causes(e.cause)
+      end
+      result
+    end
   end
-rescue
-  # Exceptions are printed to STDERR and sent to the error handler
-  # in `Racecar::Cli#run`, so we don't need to do anything here.
-  exit(1)
-else
-  exit(0)
 end
+
+# Start your engines!
+Racecar.start(ARGV)

data/extra/datadog-dashboard.json

@@ -0,0 +1 @@
+{"title":"Racecar consumer groups","description":"Dashboard for monitoring [Racecar](https://github.com/zendesk/racecar) Kafka consumer groups.","widgets":[{"id":4916208698459109,"definition":{"title":"Single-message processing","reflow_type":"fixed","type":"group","layout_type":"ordered","widgets":[{"id":82605028,"definition":{"title":"95th percentile message processing latency by topic","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"avg:racecar.consumer.process_message.latency.95percentile{$group_id,$client,$topic,$partition,$env} by {topic,group_id}","style":{"palette":"dog_classic","line_type":"solid","line_width":"thin"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":2857871641649870,"definition":{"title":"Max message processing latency by topic","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"avg:racecar.consumer.process_message.latency.max{$group_id,$client,$topic,$partition,$env} by {topic,group_id}","style":{"palette":"dog_classic","line_type":"solid","line_width":"thin"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":88579656,"definition":{"title":"Median message processing latency by topic","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"avg:racecar.consumer.process_message.latency.median{$group_id,$client,$topic,$partition,$env} by {topic}","style":{"palette":"dog_classic","line_type":"solid","line_width":"thin"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":8,"y":0,"width":4,"height":2}}]}},{"id":4068194420543030,"definition":{"title":"Batch processing","reflow_type":"fixed","type":"group","layout_type":"ordered","widgets":[{"id":341686567,"definition":{"title":"95th percentile batch processing latency by topic","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"avg:racecar.consumer.process_batch.latency.95percentile{$group_id,$client,$topic,$partition,$env} by {topic,group_id}","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":341687897,"definition":{"title":"Median batch processing latency by topic","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"avg:racecar.consumer.process_batch.latency.median{$group_id,$client,$topic,$partition,$env} by {topic}","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":5352911818003929,"definition":{"title":"Max batch processing latency by topic","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"max:racecar.consumer.process_batch.latency.max{$group_id,$client,$topic,$partition,$env} by {topic}","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":8,"y":0,"width":4,"height":2}},{"id":1654098217056312,"definition":{"title":"Max message batch size","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"max:racecar.consumer.batch_size.max{$group_id,$client,$topic,$partition,$env} by {topic}","style":{"palette":"purple","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":0,"y":2,"width":4,"height":2}},{"id":7718619791149134,"definition":{"title":"Average per-message latency in batch processing mode","show_legend":false,"legend_size":"0","legend_layout":"vertical","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"q":"max:racecar.consumer.process_batch.latency.avg{$group_id,$client,$topic,$partition,$env}/max:racecar.consumer.batch_size.avg{$group_id,$client,$topic,$partition,$env}","metadata":[{"expression":"max:racecar.consumer.process_batch.latency.avg{$env,$pod,$group_id,$client,$topic,$partition}/max:racecar.consumer.batch_size.avg{$env,$pod,$group_id,$client,$topic,$partition}","alias_name":"ms"}],"style":{"palette":"purple","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"},"markers":[]},"layout":{"x":4,"y":2,"width":4,"height":2}}]}},{"id":7110612496425151,"definition":{"title":"Throughput & Lag","reflow_type":"fixed","type":"group","layout_type":"ordered","widgets":[{"id":301212748,"definition":{"title":"Message lag changes","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"derivative(max:racecar.consumer.offset{$group_id,$client,$topic,$partition,$env} by {topic,partition,pod})","style":{"palette":"dog_classic","line_type":"solid","line_width":"thin"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":82604183,"definition":{"title":"Processing throughput by topic","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"sum:racecar.consumer.messages{$group_id,$client,$topic,$partition,$env} by {topic,group_id}.as_count()","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":5547724125706857,"definition":{"title":"Processing throughput by group","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"sum:racecar.consumer.messages{$group_id,$client,$topic,$partition,$env} by {group_id}.as_count()","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":8,"y":0,"width":4,"height":2}},{"id":375397853,"definition":{"title":"Processing throughput by host","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"sum:racecar.consumer.messages{$group_id,$client,$topic,$partition,$env} by {group_id,host}.as_rate()","style":{"palette":"dog_classic","line_type":"solid","line_width":"thin"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":0,"y":2,"width":4,"height":2}},{"id":7820607170949322,"definition":{"title":"Messages consumed in timeframe","type":"query_value","requests":[{"q":"sum:racecar.consumer.messages{$group_id,$client,$topic,$partition,$env}.as_count()","aggregator":"sum"}],"autoscale":true,"precision":0},"layout":{"x":4,"y":2,"width":4,"height":2}},{"id":1428183857213882,"definition":{"title":"Time lag (end-to-end latency)","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"avg:racecar.consumer.time_lag{$group_id,$client,$topic,$partition,$env} by {group_id,pod}","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":8,"y":2,"width":4,"height":2}}]}},{"id":1487807434456879,"definition":{"title":"Processing Errors & Group Stability","reflow_type":"fixed","type":"group","layout_type":"ordered","widgets":[{"id":82605029,"definition":{"title":"Processing errors","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"sum:racecar.consumer.process_batch.errors{$group_id,$client,$topic,$partition,$env} by {topic,pod,group_id,partition}.as_count()+sum:racecar.consumer.process_message.errors{$group_id,$client,$topic,$partition,$env} by {topic,pod,group_id,partition}.as_count()","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":83104736,"definition":{"title":"Processing error rate by topic (%)","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"(sum:racecar.consumer.process_message.errors{$group_id,$client,$topic,$partition,$env} by {topic}.as_count()/(sum:racecar.consumer.process_message.errors{$group_id,$client,$topic,$partition,$env} by {topic}.as_count()+sum:racecar.consumer.messages{$group_id,$client,$topic,$partition,$env} by {topic}.as_count()))*100","style":{"palette":"orange","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":6572534533091871,"definition":{"title":"Processing errors in timeframe","type":"query_value","requests":[{"q":"sum:racecar.consumer.process_batch.errors{$topic,$client,$group_id,$env}.as_count()+sum:racecar.consumer.process_message.errors{$topic,$client,$group_id,$env}.as_count()","aggregator":"sum","conditional_formats":[{"comparator":">","palette":"white_on_red","value":0},{"comparator":"<=","palette":"white_on_green","value":0}]}],"autoscale":true,"precision":0},"layout":{"x":8,"y":0,"width":4,"height":2}},{"id":302705923,"definition":{"title":"Pause duration","show_legend":false,"legend_size":"0","legend_layout":"vertical","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"q":"avg:racecar.consumer.pause.duration{$client,$group_id,$topic,$env} by {pod,group_id,topic,partition}","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"},"markers":[]},"layout":{"x":0,"y":2,"width":4,"height":2}},{"id":235544854,"definition":{"title":"Group joins","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"sum:racecar.consumer.join_group.count{$group_id,$client,$env} by {group_id,host}.as_count()","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":4,"y":2,"width":4,"height":2}},{"id":235544862,"definition":{"title":"Group leaves","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"sum:racecar.consumer.leave_group.count{$group_id,$client,$env} by {group_id,host}.as_count()","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":8,"y":2,"width":4,"height":2}},{"id":235545167,"definition":{"title":"Group syncs","show_legend":false,"legend_size":"0","type":"timeseries","requests":[{"q":"sum:racecar.consumer.sync_group.count{$group_id,$client,$env} by {group_id,host}.as_count()","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"}},"layout":{"x":0,"y":4,"width":4,"height":2}}]}},{"id":8013176155436939,"definition":{"title":"Producer & message delivery","reflow_type":"fixed","type":"group","layout_type":"ordered","widgets":[{"id":5948628389625057,"definition":{"title":"Message delivery latency (median)","title_size":"16","title_align":"left","show_legend":false,"type":"timeseries","requests":[{"q":"avg:racecar.producer.deliver.latency.median{$client,$env}","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"},"markers":[]},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":3158040379950811,"definition":{"title":"Producer buffer size (max)","title_size":"16","title_align":"left","show_legend":false,"type":"timeseries","requests":[{"q":"max:racecar.producer.buffer.size.max{$client,$env} by {topic}","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"},"markers":[]},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":6916375790222772,"definition":{"title":"Producer buffer size (avg) kp","title_size":"16","title_align":"left","show_legend":false,"legend_layout":"vertical","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"q":"avg:racecar.producer.buffer.size.avg{$client,$env} by {topic,host}","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"},"markers":[]},"layout":{"x":8,"y":0,"width":4,"height":2}},{"id":3160735194874896,"definition":{"title":"Message size (95p)","title_size":"16","title_align":"left","show_legend":false,"legend_layout":"vertical","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"q":"avg:racecar.producer.produce.message_size.95percentile{$env} by {topic}","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","label":"","include_zero":true,"min":"auto","max":"auto"},"markers":[]},"layout":{"x":0,"y":2,"width":4,"height":2}}]}}],"template_variables":[{"name":"env","default":"production","prefix":"env"},{"name":"group_id","default":"*","prefix":"group_id"},{"name":"client","default":"*","prefix":"client"},{"name":"topic","default":"*","prefix":"topic"},{"name":"partition","default":"*","prefix":"partition"}],"layout_type":"ordered","is_read_only":false,"notify_list":[],"reflow_type":"fixed","id":"ywc-z36-g29"}

data/lib/ensure_hash_compact.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # only needed when ruby < 2.4 and not using active support
 
 unless {}.respond_to? :compact

data/lib/generators/racecar/consumer_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module Generators
     class ConsumerGenerator < Rails::Generators::NamedBase

data/lib/generators/racecar/install_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module Generators
     class InstallGenerator < Rails::Generators::Base

data/lib/racecar/cli.rb

@@ -1,23 +1,25 @@
+# frozen_string_literal: true
+
 require "optparse"
 require "logger"
 require "fileutils"
 require "racecar/rails_config_file_loader"
 require "racecar/daemon"
+require "racecar/liveness_probe"
 
 module Racecar
   class Cli
-    def self.main(args)
-      new(args).run
+    class << self
+      def main(args)
+        new(args).run
+      end
     end
 
     def initialize(args)
       @parser = build_parser
       @parser.parse!(args)
       @consumer_name = args.first or raise Racecar::Error, "no consumer specified"
-    end
-
-    def config
-      Racecar.config
+      @runner = nil
     end
 
     def run
@@ -58,21 +60,29 @@ module Racecar
         $stderr.puts "=> Ctrl-C to shutdown consumer"
       end
 
-      processor = consumer_class.new
-
-      Racecar.run(processor)
-    rescue => e
-      $stderr.puts "=> Crashed: #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
+      if config.liveness_probe_enabled
+        $stderr.puts "=> Liveness probe enabled"
+        config.install_liveness_probe
+      end
 
-      config.error_handler.call(e)
+      processor = consumer_class.new
+      @runner = Racecar.runner(processor)
+      @runner.run
+      nil
+    end
 
-      raise
+    def stop
+      @runner.stop
     end
 
     private
 
     attr_reader :consumer_name
 
+    def config
+      Racecar.config
+    end
+
     def daemonize!
       daemon = Daemon.new(File.expand_path(config.pidfile))
 
@@ -102,12 +112,7 @@ module Racecar
         opts.on("-r", "--require STRING", "Require a library before starting the consumer") do |lib|
           $LOAD_PATH.unshift(Dir.pwd) unless load_path_modified
           load_path_modified = true
-          begin
-            require lib
-          rescue => e
-            $stderr.puts "=> #{lib} failed to load: #{e.message}"
-            exit
-          end
+          require lib
         end
 
         opts.on("-l", "--log STRING", "Log to the specified file") do |logfile|
@@ -115,13 +120,13 @@ module Racecar
         end
 
         Racecar::Config.variables.each do |variable|
-          opt_name = "--" << variable.name.to_s.gsub("_", "-")
+          opt_name = +"--#{variable.name.to_s.gsub('_', '-')}"
           opt_name << " #{variable.type.upcase}" unless variable.boolean?
 
           desc = variable.description || "N/A"
 
           if variable.default
-            desc << " (default: #{variable.default.inspect})"
+            desc += " (default: #{variable.default.inspect})"
           end
 
           opts.on(opt_name, desc) do |value|

data/lib/racecar/config.rb

@@ -1,9 +1,19 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+
 require "king_konf"
 
+require "racecar/liveness_probe"
+require "racecar/instrumenter"
+require "racecar/rebalance_listener"
+
 module Racecar
   class Config < KingKonf::Config
     env_prefix :racecar
 
+    STATISTICS_DISABLED_VALUE = 0
+
     desc "A list of Kafka brokers in the cluster that you're consuming from"
     list :brokers, default: ["localhost:9092"]
 
@@ -19,6 +29,9 @@ module Racecar
     desc "The minimum number of messages in the local consumer queue"
     integer :min_message_queue_size, default: 2000
 
+    desc "Which partition assignment strategy to use, range, roundrobin or cooperative-sticky. -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"
+    string :partition_assignment_strategy, default: "range,roundrobin"
+
     desc "Kafka consumer configuration options, separated with '=' -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"
     list :consumer, default: []
 
@@ -52,9 +65,12 @@ module Racecar
     desc "How long to wait when trying to communicate with a Kafka broker"
     float :socket_timeout, default: 30
 
-    desc "How long to allow the Kafka brokers to wait before returning messages"
+    desc "How long to allow the Kafka brokers to wait before returning messages (in seconds)"
     float :max_wait_time, default: 1
 
+    desc "How long to try to deliver a produced message before finally giving up (in seconds)"
+    float :message_timeout, default: 5*60
+
     desc "Maximum amount of data the broker shall return for a Fetch request"
     integer :max_bytes, default: 10485760
 
@@ -70,6 +86,9 @@ module Racecar
     desc "The log level for the Racecar logs"
     string :log_level, default: "info"
 
+    desc "The strategy used to determine which topic partition a message is written to when Racecar produces a value to Kafka; defaults to `consistent_random`"
+    symbol :partitioner, allowed_values: %i{consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random}, default: :consistent_random
+
     desc "Protocol used to communicate with brokers"
     symbol :security_protocol, allowed_values: %i{plaintext ssl sasl_plaintext sasl_ssl}
 
@@ -151,10 +170,33 @@ module Racecar
     desc "Whether to boot Rails when starting the consumer"
     boolean :without_rails, default: false
 
+    desc "How frequently librdkafka should report statistics to your application (in seconds). A statistics callback
+          must also be provided. This should be defined with a `statistics_callback` method on your processor. Stats
+          are disabled if this value is set to 0, or there is no callback defined. This is set by default to 1 second
+          for backward compatibility, however this can be quite memory intensive"
+    integer :statistics_interval, default: 1
+
+    desc "Whether to enable liveness probe behavior (touch the file)"
+    boolean :liveness_probe_enabled, default: false
+
+    desc "Path to a file Racecar will touch to show liveness"
+    string :liveness_probe_file_path, default: "#{Dir.tmpdir}/racecar-liveness"
+
+    desc "Used only by the liveness probe: Max time (in seconds) between liveness events before the process is considered not healthy"
+    integer :liveness_probe_max_interval, default: 5
+
     # The error handler must be set directly on the object.
     attr_reader :error_handler
 
-    attr_accessor :subscriptions, :logger
+    attr_accessor :subscriptions, :logger, :parallel_workers
+
+    def statistics_interval_ms
+      if Rdkafka::Config.statistics_callback
+        statistics_interval * 1000
+      else
+        STATISTICS_DISABLED_VALUE
+      end
+    end
 
     def max_wait_time_ms
       max_wait_time * 1000
@@ -189,6 +231,7 @@ module Racecar
     end
 
     def load_consumer_class(consumer_class)
+      self.consumer_class = consumer_class
       self.group_id = consumer_class.group_id || self.group_id
 
       self.group_id ||= [
@@ -196,13 +239,16 @@ module Racecar
         group_id_prefix,
 
         # MyFunnyConsumer => my-funny-consumer
-        consumer_class.name.gsub(/[a-z][A-Z]/) {|str| str[0] << "-" << str[1] }.downcase,
-      ].compact.join("")
+        consumer_class.name.gsub(/[a-z][A-Z]/) { |str| "#{str[0]}-#{str[1]}" }.downcase,
+      ].compact.join
 
+      self.parallel_workers = consumer_class.parallel_workers
       self.subscriptions = consumer_class.subscriptions
       self.max_wait_time = consumer_class.max_wait_time || self.max_wait_time
+      self.fetch_messages = consumer_class.fetch_messages || self.fetch_messages
       self.pidfile ||= "#{group_id}.pid"
     end
+    attr_accessor :consumer_class
 
     def on_error(&handler)
       @error_handler = handler
@@ -224,11 +270,41 @@ module Racecar
       producer_config
     end
 
+    def instrumenter
+      @instrumenter ||= begin
+        default_payload = { client_id: client_id, group_id: group_id }
+
+        if defined?(ActiveSupport::Notifications)
+          # ActiveSupport needs `concurrent-ruby` but doesn't `require` it.
+          require 'concurrent/utility/monotonic_time'
+          Instrumenter.new(backend: ActiveSupport::Notifications, default_payload: default_payload)
+        else
+          logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
+          NullInstrumenter
+        end
+      end
+    end
+    attr_writer :instrumenter
+
+    def install_liveness_probe
+      liveness_probe.tap(&:install)
+    end
+
+    def liveness_probe
+      require "active_support/notifications"
+      @liveness_probe ||= LivenessProbe.new(
+        ActiveSupport::Notifications,
+        liveness_probe_file_path,
+        liveness_probe_max_interval
+      )
+    end
+
     private
 
     def rdkafka_security_config
       {
         "security.protocol" => security_protocol,
+        "enable.ssl.certificate.verification" => ssl_verify_hostname,
         "ssl.ca.location" => ssl_ca_location,
         "ssl.crl.location" => ssl_crl_location,
         "ssl.keystore.location" => ssl_keystore_location,

data/lib/racecar/consumer.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+require "racecar/message_delivery_error"
+
 module Racecar
   class Consumer
     Subscription = Struct.new(:topic, :start_from_beginning, :max_bytes_per_partition, :additional_config)
@@ -5,7 +9,7 @@ module Racecar
     class << self
       attr_accessor :max_wait_time
       attr_accessor :group_id
-      attr_accessor :producer, :consumer
+      attr_accessor :producer, :consumer, :parallel_workers, :fetch_messages
 
       def subscriptions
         @subscriptions ||= []
@@ -23,29 +27,68 @@ module Racecar
       # @param additional_config [Hash] Configuration properties for consumer.
       #   See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       # @return [nil]
-      def subscribes_to(*topics, start_from_beginning: true, max_bytes_per_partition: 1048576, additional_config: {})
+      def subscribes_to(
+        *topics,
+        start_from_beginning: true,
+        max_bytes_per_partition: 1048576,
+        additional_config: {}
+      )
         topics.each do |topic|
           subscriptions << Subscription.new(topic, start_from_beginning, max_bytes_per_partition, additional_config)
         end
       end
+
+      # Rebalance hooks for subclasses to override
+      def on_partitions_assigned(rebalance_event); end
+      def on_partitions_revoked(rebalance_event); end
     end
 
-    def configure(producer:, consumer:, instrumenter: NullInstrumenter)
+    def configure(producer:, consumer:, instrumenter: NullInstrumenter, config: Racecar.config)
       @producer = producer
+      @delivery_handles = []
+
       @consumer = consumer
+
       @instrumenter = instrumenter
+      @config = config
     end
 
     def teardown; end
 
-    # Delivers messages that got produced.
+    # Blocks until all messages produced so far have been successfully published. If
+    # message delivery finally fails, a Racecar::MessageDeliveryError is raised. The
+    # delivery failed for the reason in the exception. The error can be broker side
+    # (e.g. downtime, configuration issue) or specific to the message being sent. The
+    # caller must handle the latter cases or run into head of line blocking.
     def deliver!
       @delivery_handles ||= []
       if @delivery_handles.any?
         instrumentation_payload = { delivered_message_count: @delivery_handles.size }
 
         @instrumenter.instrument('deliver_messages', instrumentation_payload) do
-          @delivery_handles.each(&:wait)
+          @delivery_handles.each do |handle|
+            begin
+              # rdkafka-ruby checks every wait_timeout seconds if the message was
+              # successfully delivered, up to max_wait_timeout seconds before raising
+              # Rdkafka::AbstractHandle::WaitTimeoutError. librdkafka will (re)try to
+              # deliver all messages in the background, until "config.message_timeout"
+              # (message.timeout.ms) is exceeded. Phrased differently, rdkafka-ruby's
+              # WaitTimeoutError is just informative.
+              # The raising can be avoided if max_wait_timeout below is greater than
+              # config.message_timeout, but config is not available here (without
+              # changing the interface).
+              handle.wait(max_wait_timeout: 60, wait_timeout: 0.1)
+            rescue Rdkafka::AbstractHandle::WaitTimeoutError => e
+              partition = MessageDeliveryError.partition_from_delivery_handle(handle)
+              # ideally we could use the logger passed to the Runner, but it is not
+              # available here. The runner sets it for Rdkafka, though, so we can use
+              # that instead.
+              @config.logger.debug "Still trying to deliver message to (partition #{partition})... (will try up to Racecar.config.message_timeout)"
+              retry
+            rescue Rdkafka::RdkafkaError => e
+              raise MessageDeliveryError.new(e, handle)
+            end
+          end
         end
       end
       @delivery_handles.clear
@@ -54,13 +97,14 @@ module Racecar
     protected
 
     # https://github.com/appsignal/rdkafka-ruby#producing-messages
-    def produce(payload, topic:, key:, partition_key: nil, headers: nil, create_time: nil)
+    def produce(payload, topic:, key: nil, partition: nil, partition_key: nil, headers: nil, create_time: nil)
       @delivery_handles ||= []
       message_size = payload.respond_to?(:bytesize) ? payload.bytesize : 0
       instrumentation_payload = {
         value: payload,
         headers: headers,
         key: key,
+        partition: partition,
         partition_key: partition_key,
         topic: topic,
         message_size: message_size,
@@ -73,6 +117,7 @@ module Racecar
           topic: topic,
           payload: payload,
           key: key,
+          partition: partition,
           partition_key: partition_key,
           timestamp: create_time,
           headers: headers,