racecar 2.0.0 → 2.10.0.beta2

data/lib/racecar/consumer_set.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   class ConsumerSet
     MAX_POLL_TRIES = 10
@@ -10,53 +12,50 @@ module Racecar
       @consumers = []
       @consumer_id_iterator = (0...@config.subscriptions.size).cycle
 
+      @previous_retries = 0
+
       @last_poll_read_nil_message = false
+      @paused_tpls = Hash.new { |h, k| h[k] = {} }
     end
 
-    def poll(timeout_ms)
-      maybe_select_next_consumer
-      started_at ||= Time.now
-      try ||= 0
-      remain ||= timeout_ms
-
-      msg = remain <= 0 ? nil : current.poll(remain)
-    rescue Rdkafka::RdkafkaError => e
-      wait_before_retry_ms = 100 * (2**try) # 100ms, 200ms, 400ms, …
-      try += 1
-      raise if try >= MAX_POLL_TRIES || remain <= wait_before_retry_ms
-
-      @logger.error "(try #{try}): Error for topic subscription #{current_subscription}: #{e}"
-
-      case e.code
-      when :max_poll_exceeded, :transport # -147, -195
-        reset_current_consumer
-      end
-
-      remain = remaining_time_ms(timeout_ms, started_at)
-      raise if remain <= wait_before_retry_ms
-
-      sleep wait_before_retry_ms/1000.0
-      retry
-    ensure
-      @last_poll_read_nil_message = true if msg.nil?
+    def poll(max_wait_time_ms = @config.max_wait_time_ms)
+      batch_poll(max_wait_time_ms, 1).first
     end
 
-    # XXX: messages are not guaranteed to be from the same partition
-    def batch_poll(timeout_ms)
-      @batch_started_at = Time.now
-      @messages = []
-      while collect_messages_for_batch? do
-        remain = remaining_time_ms(timeout_ms, @batch_started_at)
-        break if remain <= 0
-        msg = poll(remain)
+    # batch_poll collects messages until any of the following occurs:
+    # - max_wait_time_ms time has passed
+    # - max_messages have been collected
+    # - a nil message was polled (end of topic, Kafka stalled, etc.)
+    #
+    # The messages are from a single topic, but potentially from more than one partition.
+    #
+    # Any errors during polling are retried in an exponential backoff fashion. If an error
+    # occurs, but there is no time left for a backoff and retry, it will return the
+    # already collected messages and only retry on the next call.
+    def batch_poll(max_wait_time_ms = @config.max_wait_time_ms, max_messages = @config.fetch_messages)
+      started_at = Time.now
+      remain_ms = max_wait_time_ms
+      maybe_select_next_consumer
+      messages = []
+
+      while remain_ms > 0 && messages.size < max_messages
+        remain_ms = remaining_time_ms(max_wait_time_ms, started_at)
+        msg = poll_with_retries(remain_ms)
         break if msg.nil?
-        @messages << msg
+        messages << msg
       end
-      @messages
+
+      messages
     end
 
     def store_offset(message)
       current.store_offset(message)
+    rescue Rdkafka::RdkafkaError => e
+      if e.code == :state # -172
+        @logger.warn "Attempted to store_offset, but we're not subscribed to it: #{ErroneousStateError.new(e)}"
+        return
+      end
+      raise e
     end
 
     def commit
@@ -67,11 +66,17 @@ module Racecar
 
     def close
       each_subscribed(&:close)
+      @paused_tpls.clear
     end
 
     def current
       @consumers[@consumer_id_iterator.peek] ||= begin
-        consumer = Rdkafka::Config.new(rdkafka_config(current_subscription)).consumer
+        consumer_config = Rdkafka::Config.new(rdkafka_config(current_subscription))
+        listener = RebalanceListener.new(@config.consumer_class, @instrumenter)
+        consumer_config.consumer_rebalance_listener = listener
+        consumer = consumer_config.consumer
+        listener.rdkafka_consumer = consumer
+
         @instrumenter.instrument('join_group') do
           consumer.subscribe current_subscription.topic
         end
@@ -97,16 +102,25 @@ module Racecar
       consumer.pause(filtered_tpl)
       fake_msg = OpenStruct.new(topic: topic, partition: partition, offset: offset)
       consumer.seek(fake_msg)
+
+      @paused_tpls[topic][partition] = [consumer, filtered_tpl]
     end
 
     def resume(topic, partition)
       consumer, filtered_tpl = find_consumer_by(topic, partition)
+
+      if !consumer && @paused_tpls[topic][partition]
+        consumer, filtered_tpl = @paused_tpls[topic][partition]
+      end
+
       if !consumer
         @logger.info "Attempted to resume #{topic}/#{partition}, but we're not subscribed to it"
         return
       end
 
       consumer.resume(filtered_tpl)
+      @paused_tpls[topic].delete(partition)
+      @paused_tpls.delete(topic) if @paused_tpls[topic].empty?
     end
 
     alias :each :each_subscribed
@@ -123,6 +137,55 @@ module Racecar
 
     private
 
+    # polls a single message from the current consumer, retrying errors with exponential
+    # backoff. The sleep time is capped by max_wait_time_ms. If there's enough time budget
+    # left, it will retry before returning. If there isn't, the retry will only occur on
+    # the next call. It tries up to MAX_POLL_TRIES before passing on the exception.
+    def poll_with_retries(max_wait_time_ms)
+      try ||= @previous_retries
+      @previous_retries = 0
+      started_at ||= Time.now
+      remain_ms = remaining_time_ms(max_wait_time_ms, started_at)
+
+      wait_ms = try == 0 ? 0 : 50 * (2**try) # 0ms, 100ms, 200ms, 400ms, …
+      if wait_ms >= max_wait_time_ms && remain_ms > 1
+        @logger.debug "Capping #{wait_ms}ms to #{max_wait_time_ms-1}ms."
+        sleep (max_wait_time_ms-1)/1000.0
+        remain_ms = 1
+      elsif try == 0 && remain_ms == 0
+        @logger.debug "No time remains for polling messages. Will try on next call."
+        return nil
+      elsif wait_ms >= remain_ms
+        @logger.warn "Only #{remain_ms}ms left, but want to wait for #{wait_ms}ms before poll. Will retry on next call."
+        @previous_retries = try
+        return nil
+      elsif wait_ms > 0
+        sleep wait_ms/1000.0
+        remain_ms -= wait_ms
+      end
+
+      poll_current_consumer(remain_ms)
+    rescue Rdkafka::RdkafkaError => e
+      try += 1
+      @instrumenter.instrument("poll_retry", try: try, rdkafka_time_limit: remain_ms, exception: e)
+      @logger.error "(try #{try}/#{MAX_POLL_TRIES}): Error for topic subscription #{current_subscription}: #{e}"
+      raise if try >= MAX_POLL_TRIES
+      retry
+    end
+
+    # polls a message for the current consumer, handling any API edge cases.
+    def poll_current_consumer(max_wait_time_ms)
+      msg = current.poll(max_wait_time_ms)
+    rescue Rdkafka::RdkafkaError => e
+      case e.code
+      when :max_poll_exceeded, :transport, :not_coordinator # -147, -195, 16
+        reset_current_consumer
+      end
+      raise
+    ensure
+      @last_poll_read_nil_message = msg.nil?
+    end
+
     def find_consumer_by(topic, partition)
       each do |consumer|
         tpl = consumer.assignment.to_h
@@ -140,7 +203,12 @@ module Racecar
     end
 
     def reset_current_consumer
-      @consumers[@consumer_id_iterator.peek] = nil
+      current_consumer_id = @consumer_id_iterator.peek
+      @logger.info "Resetting consumer with id: #{current_consumer_id}"
+
+      consumer = @consumers[current_consumer_id]
+      consumer.close unless consumer.nil?
+      @consumers[current_consumer_id] = nil
     end
 
     def maybe_select_next_consumer
@@ -160,14 +228,14 @@ module Racecar
       @logger.debug "Nothing to commit."
     end
 
-    def collect_messages_for_batch?
-      @messages.size < @config.fetch_messages &&
-      (Time.now - @batch_started_at) < @config.max_wait_time
-    end
-
     def rdkafka_config(subscription)
       # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       config = {
+        # Manually store offset after messages have been processed successfully
+        # to avoid marking failed messages as committed. The call just updates
+        # a value within librdkafka and is asynchronously written to proper
+        # storage through auto commits.
+        "enable.auto.offset.store" => false,
         "auto.commit.interval.ms" => @config.offset_commit_interval * 1000,
         "auto.offset.reset"       => subscription.start_from_beginning ? "earliest" : "largest",
         "bootstrap.servers"       => @config.brokers.join(","),
@@ -183,7 +251,8 @@ module Racecar
         "queued.min.messages"     => @config.min_message_queue_size,
         "session.timeout.ms"      => @config.session_timeout * 1000,
         "socket.timeout.ms"       => @config.socket_timeout * 1000,
-        "statistics.interval.ms"  => 1000, # 1s is the highest granularity offered
+        "statistics.interval.ms"  => @config.statistics_interval_ms,
+        "partition.assignment.strategy" => @config.partition_assignment_strategy,
       }
       config.merge! @config.rdkafka_consumer
       config.merge! subscription.additional_config

data/lib/racecar/ctl.rb

@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 require "optparse"
 require "racecar/rails_config_file_loader"
 require "racecar/daemon"
+require "racecar/message_delivery_error"
 
 module Racecar
   class Ctl
@@ -29,6 +32,21 @@ module Racecar
       @command = command
     end
 
+    def liveness_probe(args)
+      require "racecar/liveness_probe"
+      parse_options!(args)
+
+      if ENV["RAILS_ENV"] && File.exist?("config/racecar.yml")
+        Racecar.config.load_file("config/racecar.yml", ENV["RAILS_ENV"])
+      end
+
+      if File.exist?("config/racecar.rb")
+        require "./config/racecar"
+      end
+
+      Racecar.config.liveness_probe.check_liveness_within_interval!
+    end
+
     def status(args)
       parse_options!(args)
 
@@ -94,11 +112,17 @@ module Racecar
       Racecar.config.validate!
 
       producer = Rdkafka::Config.new({
-        "bootstrap.servers": Racecar.config.brokers.join(","),
-        "client.id":         Racecar.config.client_id,
+        "bootstrap.servers":  Racecar.config.brokers.join(","),
+        "client.id":          Racecar.config.client_id,
+        "message.timeout.ms": Racecar.config.message_timeout * 1000,
       }.merge(Racecar.config.rdkafka_producer)).producer
 
-      producer.produce(payload: message.value, key: message.key, topic: message.topic).wait
+      handle = producer.produce(payload: message.value, key: message.key, topic: message.topic)
+      begin
+        handle.wait(max_wait_timeout: Racecar.config.message_timeout)
+      rescue Rdkafka::RdkafkaError => e
+        raise MessageDeliveryError.new(e, handle)
+      end
 
       $stderr.puts "=> Delivered message to Kafka cluster"
     end
@@ -116,5 +140,9 @@ module Racecar
 
       parser.parse!(args)
     end
+
+    def config
+      Racecar.config
+    end
   end
 end

data/lib/racecar/daemon.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   class Daemon
     attr_reader :pidfile
@@ -52,7 +54,7 @@ module Racecar
     end
 
     def pid
-      if File.exists?(pidfile)
+      if File.exist?(pidfile)
         File.read(pidfile).to_i
       else
         nil
@@ -87,7 +89,7 @@ module Racecar
       end
 
       at_exit do
-        File.delete(pidfile) if File.exists?(pidfile)
+        File.delete(pidfile) if File.exist?(pidfile)
       end
     rescue Errno::EEXIST
       check_pid

data/lib/racecar/datadog.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 begin
   require "datadog/statsd"
 rescue LoadError
@@ -61,10 +63,14 @@ module Racecar
         clear
       end
 
+      def close
+        @statsd&.close
+      end
+
       private
 
       def clear
-        @statsd && @statsd.close
+        close
         @statsd = nil
       end
     end
@@ -73,8 +79,8 @@ module Racecar
       private
 
       %w[increment histogram count timing gauge].each do |type|
-        define_method(type) do |*args|
-          emit(type, *args)
+        define_method(type) do |*args, **kwargs|
+          emit(type, *args, **kwargs)
         end
       end
 
@@ -155,6 +161,15 @@ module Racecar
         end
       end
 
+      def poll_retry(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+        }
+        rdkafka_error_code = event.payload.fetch(:exception).code.to_s.gsub(/\W/, '')
+        increment("consumer.poll.rdkafka_error.#{rdkafka_error_code}", tags: tags)
+      end
+
       def main_loop(event)
         tags = {
           client: event.payload.fetch(:client_id),
@@ -196,6 +211,10 @@ module Racecar
           topic: topic,
         }
 
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
         # This gets us the write rate.
         increment("producer.produce.messages", tags: tags.merge(topic: topic))
 
@@ -230,6 +249,67 @@ module Racecar
         increment("producer.ack.messages", tags: tags)
       end
 
+      def produce_delivery_error(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+        }
+
+        increment("producer.produce.delivery.errors", tags: tags)
+      end
+
+      def produce_async(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+        buffer_size = event.payload.fetch(:buffer_size)
+
+        tags = {
+          client: client,
+          topic: topic,
+        }
+
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
+        # This gets us the write rate.
+        increment("producer.produce.messages", tags: tags.merge(topic: topic))
+
+        # Information about typical/average/95p message size.
+        histogram("producer.produce.message_size", message_size, tags: tags.merge(topic: topic))
+
+        # Aggregate message size.
+        count("producer.produce.message_size.sum", message_size, tags: tags.merge(topic: topic))
+
+        # This gets us the avg/max buffer size per producer.
+        histogram("producer.buffer.size", buffer_size, tags: tags)
+      end
+
+      def produce_sync(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+
+        tags = {
+          client: client,
+          topic: topic,
+        }
+
+        if event.payload.key?(:exception)
+          increment("producer.produce.errors", tags: tags)
+        end
+
+
+        # This gets us the write rate.
+        increment("producer.produce.messages", tags: tags.merge(topic: topic))
+
+        # Information about typical/average/95p message size.
+        histogram("producer.produce.message_size", message_size, tags: tags.merge(topic: topic))
+
+        # Aggregate message size.
+        count("producer.produce.message_size.sum", message_size, tags: tags.merge(topic: topic))
+      end
+
       attach_to "racecar"
     end
   end

data/lib/racecar/delivery_callback.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Racecar
+  class DeliveryCallback
+    attr_reader :instrumenter
+
+    def initialize(instrumenter:)
+      @instrumenter = instrumenter
+    end
+
+    def call(delivery_report)
+      if delivery_report.error.to_i.zero?
+        payload = {
+          offset: delivery_report.offset,
+          partition: delivery_report.partition
+        }
+        instrumenter.instrument("acknowledged_message", payload)
+      else
+        payload = {
+          partition: delivery_report.partition,
+          exception: delivery_report.error
+        }
+        instrumenter.instrument("produce_delivery_error", payload)
+      end
+    end
+  end
+end

data/lib/racecar/erroneous_state_error.rb

@@ -0,0 +1,34 @@
+# `rd_kafka_offsets_store()` (et.al) returns an error for any
+# partition that is not currently assigned (through `rd_kafka_*assign()`).
+# This prevents a race condition where an application would store offsets
+# after the assigned partitions had been revoked (which resets the stored
+# offset), that could cause these old stored offsets to be committed later
+# when the same partitions were assigned to this consumer again - effectively
+# overwriting any committed offsets by any consumers that were assigned the
+# same partitions previously. This would typically result in the offsets
+# rewinding and messages to be reprocessed.
+# As an extra effort to avoid this situation the stored offset is now
+# also reset when partitions are assigned (through `rd_kafka_*assign()`).
+module Racecar
+  class ErroneousStateError < StandardError
+    def initialize(rdkafka_error)
+      raise rdkafka_error unless rdkafka_error.is_a?(Rdkafka::RdkafkaError)
+
+      @rdkafka_error = rdkafka_error
+    end
+
+    attr_reader :rdkafka_error
+
+    def code
+      @rdkafka_error.code
+    end
+
+    def to_s
+      <<~EOM
+        Partition is no longer assigned to this consumer and the offset could not be stored for commit.
+        #{@rdkafka_error.to_s}
+      EOM
+    end
+
+  end
+end

data/lib/racecar/heroku.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'tempfile'
+
+# Heroku Kafka addon provides 4 ENVs to connect to their Kafka Broker
+# KAFKA_TRUSTED_CERT, KAFKA_CLIENT_CERT, KAFKA_CLIENT_CERT_KEY, KAFKA_URL
+# This will work only if the Heroku Kafka add-on is aliased to "KAFKA"
+
+$stderr.puts "=> Loading configuration from Heroku Kafka ENVs"
+
+module Racecar
+  module Heroku
+    def self.load_configuration!
+      [
+        "KAFKA_URL",
+        "KAFKA_TRUSTED_CERT",
+        "KAFKA_CLIENT_CERT",
+        "KAFKA_CLIENT_CERT_KEY"
+      ]. each do |env_name|
+        if ENV[env_name].nil?
+          $stderr.puts "Error: ENV #{env_name} is not set"
+          exit 1
+        end
+      end
+
+      Racecar.configure do |config|
+        ca_cert = ENV["KAFKA_TRUSTED_CERT"]
+        client_cert = ENV["KAFKA_CLIENT_CERT"]
+        client_cert_key = ENV["KAFKA_CLIENT_CERT_KEY"]
+
+        tmp_file_path = lambda do |data|
+          tempfile = Tempfile.new(['', '.pem'])
+          tempfile << data
+          tempfile.close
+          tempfile.path
+        end
+
+        config.security_protocol = :ssl
+        config.ssl_ca_location = tmp_file_path.call(ca_cert)
+        config.ssl_certificate_location = tmp_file_path.call(client_cert)
+        config.ssl_key_location = tmp_file_path.call(client_cert_key)
+
+        config.brokers = ENV["KAFKA_URL"].to_s.gsub('kafka+ssl://', '').split(',')
+      end
+    end
+  end
+end
+
+Racecar::Heroku.load_configuration!

data/lib/racecar/instrumenter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   ##
   # Common API for instrumentation to standardize
@@ -7,14 +9,9 @@ module Racecar
     NAMESPACE = "racecar"
     attr_reader :backend
 
-    def initialize(default_payload = {})
+    def initialize(backend:, default_payload: {})
+      @backend = backend
       @default_payload = default_payload
-
-      @backend = if defined?(ActiveSupport::Notifications)
-        ActiveSupport::Notifications
-      else
-        NullInstrumenter
-      end
     end
 
     def instrument(event_name, payload = {}, &block)

data/lib/racecar/liveness_probe.rb

@@ -0,0 +1,78 @@
+require "fileutils"
+
+module Racecar
+  class LivenessProbe
+    def initialize(message_bus, file_path, max_interval)
+      @message_bus = message_bus
+      @file_path = file_path
+      @max_interval = max_interval
+      @subscribers = []
+    end
+
+    attr_reader :message_bus, :file_path, :max_interval, :subscribers
+    private     :message_bus, :file_path, :max_interval, :subscribers
+
+    def check_liveness_within_interval!
+      unless liveness_event_within_interval?
+        $stderr.puts "Racecar healthcheck failed: No liveness within interval #{max_interval}s. Last liveness at #{last_liveness_event_at}, #{elapsed_since_liveness_event} seconds ago."
+        Process.exit(1)
+      end
+    end
+
+    def liveness_event_within_interval?
+      elapsed_since_liveness_event < max_interval
+    rescue Errno::ENOENT
+      $stderr.puts "Racecar healthcheck failed: Liveness file not found `#{file_path}`"
+      Process.exit(1)
+    end
+
+    def install
+      unless file_path && file_writeable?
+        raise(
+          "Liveness probe configuration error: `liveness_probe_file_path` must be set to a writable file path.\n" \
+            "  Set `RACECAR_LIVENESS_PROBE_FILE_PATH` and `RACECAR_LIVENESS_MAX_INTERVAL` environment variables."
+        )
+      end
+
+      subscribers << message_bus.subscribe("start_main_loop.racecar") do
+        touch_liveness_file
+      end
+
+      subscribers = message_bus.subscribe("shut_down.racecar") do
+        delete_liveness_file
+      end
+
+      nil
+    end
+
+    def uninstall
+      subscribers.each { |s| message_bus.unsubscribe(s) }
+    end
+
+    private
+
+    def elapsed_since_liveness_event
+      Time.now - last_liveness_event_at
+    end
+
+    def last_liveness_event_at
+      File.mtime(file_path)
+    end
+
+    def touch_liveness_file
+      FileUtils.touch(file_path)
+    end
+
+    def delete_liveness_file
+      FileUtils.rm_rf(file_path)
+    end
+
+    def file_writeable?
+      File.write(file_path, "")
+      File.unlink(file_path)
+      true
+    rescue
+      false
+    end
+  end
+end

data/lib/racecar/message.rb

@@ -1,11 +1,16 @@
+# frozen_string_literal: true
+
 require "forwardable"
 
 module Racecar
   class Message
     extend Forwardable
 
-    def initialize(rdkafka_message)
+    attr_reader :retries_count
+
+    def initialize(rdkafka_message, retries_count: nil)
       @rdkafka_message = rdkafka_message
+      @retries_count   = retries_count
     end
 
     def_delegators :@rdkafka_message, :topic, :partition, :offset, :key, :headers