racecar 0.5.0.beta2 → 2.2.0

data/lib/racecar/instrumenter.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Racecar
+  ##
+  # Common API for instrumentation to standardize
+  # namespace and default payload
+  #
+  class Instrumenter
+    NAMESPACE = "racecar"
+    attr_reader :backend
+
+    def initialize(default_payload = {})
+      @default_payload = default_payload
+
+      @backend = if defined?(ActiveSupport::Notifications)
+        # ActiveSupport needs `concurrent-ruby` but doesn't `require` it.
+        require 'concurrent/utility/monotonic_time'
+        ActiveSupport::Notifications
+      else
+        NullInstrumenter
+      end
+    end
+
+    def instrument(event_name, payload = {}, &block)
+      @backend.instrument("#{event_name}.#{NAMESPACE}", @default_payload.merge(payload), &block)
+    end
+  end
+end

data/lib/racecar/message.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Racecar
+  class Message
+    extend Forwardable
+
+    attr_reader :retries_count
+
+    def initialize(rdkafka_message, retries_count: nil)
+      @rdkafka_message = rdkafka_message
+      @retries_count   = retries_count
+    end
+
+    def_delegators :@rdkafka_message, :topic, :partition, :offset, :key, :headers
+
+    def value
+      @rdkafka_message.payload
+    end
+
+    def create_time
+      @rdkafka_message.timestamp
+    end
+
+    def ==(other)
+      @rdkafka_message == other.instance_variable_get(:@rdkafka_message)
+    end
+  end
+end

data/lib/racecar/null_instrumenter.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Racecar
+  # Ignores all instrumentation events.
+  class NullInstrumenter
+    def self.instrument(*)
+      yield({}) if block_given?
+    end
+  end
+end

data/lib/racecar/pause.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Racecar
+  class Pause
+    attr_reader :pauses_count
+
+    def initialize(timeout: nil, max_timeout: nil, exponential_backoff: false)
+      @started_at = nil
+      @pauses_count = 0
+      @timeout = timeout
+      @max_timeout = max_timeout
+      @exponential_backoff = exponential_backoff
+    end
+
+    def pause!
+      @started_at = Time.now
+      @ends_at = @started_at + backoff_interval unless @timeout.nil?
+      @pauses_count += 1
+    end
+
+    def resume!
+      @started_at = nil
+      @ends_at = nil
+    end
+
+    def paused?
+      !@started_at.nil?
+    end
+
+    def pause_duration
+      if paused?
+        Time.now - @started_at
+      else
+        0
+      end
+    end
+
+    def expired?
+      return false if @timeout.nil?
+      return true unless @ends_at
+      Time.now >= @ends_at
+    end
+
+    def reset!
+      @pauses_count = 0
+    end
+
+    def backoff_interval
+      return Float::INFINITY if @timeout.nil?
+
+      backoff_factor = @exponential_backoff ? 2**@pauses_count : 1
+      timeout = backoff_factor * @timeout
+
+      timeout = @max_timeout if @max_timeout && timeout > @max_timeout
+
+      timeout
+    end
+  end
+end

data/lib/racecar/rails_config_file_loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module RailsConfigFileLoader
     def self.load!

data/lib/racecar/runner.rb

@@ -1,148 +1,257 @@
-require "kafka"
+# frozen_string_literal: true
+
+require "rdkafka"
+require "racecar/pause"
+require "racecar/message"
 
 module Racecar
   class Runner
-    attr_reader :processor, :config, :logger, :consumer
+    attr_reader :processor, :config, :logger
 
     def initialize(processor, config:, logger:, instrumenter: NullInstrumenter)
       @processor, @config, @logger = processor, config, logger
       @instrumenter = instrumenter
+      @stop_requested = false
+      Rdkafka::Config.logger = logger
+
+      if processor.respond_to?(:statistics_callback)
+        Rdkafka::Config.statistics_callback = processor.method(:statistics_callback).to_proc
+      end
+
+      setup_pauses
     end
 
-    def stop
-      Thread.new do
-        processor.teardown
-        consumer.stop unless consumer.nil?
-      end.join
+    def setup_pauses
+      timeout = if config.pause_timeout == -1
+        nil
+      elsif config.pause_timeout == 0
+        # no op, handled elsewhere
+      elsif config.pause_timeout > 0
+        config.pause_timeout
+      else
+        raise ArgumentError, "Invalid value for pause_timeout: must be integer greater or equal -1"
+      end
+
+      @pauses = Hash.new {|h, k|
+        h[k] = Hash.new {|h2, k2|
+          h2[k2] = ::Racecar::Pause.new(
+            timeout:             timeout,
+            max_timeout:         config.max_pause_timeout,
+            exponential_backoff: config.pause_with_exponential_backoff
+          )
+        }
+      }
     end
 
     def run
-      kafka = Kafka.new(
-        client_id: config.client_id,
-        seed_brokers: config.brokers,
-        logger: logger,
-        connect_timeout: config.connect_timeout,
-        socket_timeout: config.socket_timeout,
-        ssl_ca_cert: config.ssl_ca_cert,
-        ssl_ca_cert_file_path: config.ssl_ca_cert_file_path,
-        ssl_client_cert: config.ssl_client_cert,
-        ssl_client_cert_key: config.ssl_client_cert_key,
-        sasl_plain_username: config.sasl_plain_username,
-        sasl_plain_password: config.sasl_plain_password,
-        sasl_scram_username: config.sasl_scram_username,
-        sasl_scram_password: config.sasl_scram_password,
-        sasl_scram_mechanism: config.sasl_scram_mechanism,
-        sasl_over_ssl: config.sasl_over_ssl,
-        ssl_ca_certs_from_system: config.ssl_ca_certs_from_system,
-      )
+      install_signal_handlers
+      @stop_requested = false
 
-      @consumer = kafka.consumer(
-        group_id: config.group_id,
-        offset_commit_interval: config.offset_commit_interval,
-        offset_commit_threshold: config.offset_commit_threshold,
-        session_timeout: config.session_timeout,
-        heartbeat_interval: config.heartbeat_interval,
-        offset_retention_time: config.offset_retention_time,
-        fetcher_max_queue_size: config.max_fetch_queue_size,
+      # Configure the consumer with a producer so it can produce messages and
+      # with a consumer so that it can support advanced use-cases.
+      processor.configure(
+        producer:     producer,
+        consumer:     consumer,
+        instrumenter: @instrumenter,
       )
 
+      instrumentation_payload = {
+        consumer_class: processor.class.to_s,
+        consumer_set: consumer
+      }
+
+      # Main loop
+      loop do
+        break if @stop_requested
+        resume_paused_partitions
+        @instrumenter.instrument("main_loop", instrumentation_payload) do
+          case process_method
+          when :batch then
+            msg_per_part = consumer.batch_poll(config.max_wait_time_ms).group_by(&:partition)
+            msg_per_part.each_value do |messages|
+              process_batch(messages)
+            end
+          when :single then
+            message = consumer.poll(config.max_wait_time_ms)
+            process(message) if message
+          end
+        end
+      end
+
+      logger.info "Gracefully shutting down"
+      processor.deliver!
+      processor.teardown
+      consumer.commit
+      @instrumenter.instrument('leave_group') do
+        consumer.close
+      end
+    end
+
+    def stop
+      @stop_requested = true
+    end
+
+    private
+
+    attr_reader :pauses
+
+    def process_method
+      @process_method ||= begin
+        case
+        when processor.respond_to?(:process_batch) then :batch
+        when processor.respond_to?(:process) then :single
+        else
+          raise NotImplementedError, "Consumer class must implement process or process_batch method"
+        end
+      end
+    end
+
+    def consumer
+      @consumer ||= begin
+        # Manually store offset after messages have been processed successfully
+        # to avoid marking failed messages as committed. The call just updates
+        # a value within librdkafka and is asynchronously written to proper
+        # storage through auto commits.
+        config.consumer << "enable.auto.offset.store=false"
+        ConsumerSet.new(config, logger, @instrumenter)
+      end
+    end
+
+    def producer
+      @producer ||= Rdkafka::Config.new(producer_config).producer.tap do |producer|
+        producer.delivery_callback = delivery_callback
+      end
+    end
+
+    def producer_config
+      # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+      producer_config = {
+        "bootstrap.servers"      => config.brokers.join(","),
+        "client.id"              => config.client_id,
+        "statistics.interval.ms" => 1000,
+      }
+      producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
+      producer_config.merge!(config.rdkafka_producer)
+      producer_config
+    end
+
+    def delivery_callback
+      ->(delivery_report) do
+        payload = {
+          offset: delivery_report.offset,
+          partition: delivery_report.partition
+        }
+        @instrumenter.instrument("acknowledged_message", payload)
+      end
+    end
+
+    def install_signal_handlers
       # Stop the consumer on SIGINT, SIGQUIT or SIGTERM.
       trap("QUIT") { stop }
-      trap("INT") { stop }
+      trap("INT")  { stop }
       trap("TERM") { stop }
 
       # Print the consumer config to STDERR on USR1.
       trap("USR1") { $stderr.puts config.inspect }
+    end
 
-      config.subscriptions.each do |subscription|
-        consumer.subscribe(
-          subscription.topic,
-          start_from_beginning: subscription.start_from_beginning,
-          max_bytes_per_partition: subscription.max_bytes_per_partition,
-        )
-      end
+    def process(message)
+      instrumentation_payload = {
+        consumer_class: processor.class.to_s,
+        topic:          message.topic,
+        partition:      message.partition,
+        offset:         message.offset,
+        create_time:    message.timestamp,
+        key:            message.key,
+        value:          message.payload,
+        headers:        message.headers
+      }
 
-      # Configure the consumer with a producer so it can produce messages.
-      producer = kafka.producer(
-        compression_codec: config.producer_compression_codec,
-      )
+      @instrumenter.instrument("start_process_message", instrumentation_payload)
+      with_pause(message.topic, message.partition, message.offset..message.offset) do |pause|
+        begin
+          @instrumenter.instrument("process_message", instrumentation_payload) do
+            processor.process(Racecar::Message.new(message, retries_count: pause.pauses_count))
+            processor.deliver!
+            consumer.store_offset(message)
+          end
+        rescue => e
+          instrumentation_payload[:retries_count] = pause.pauses_count
+          config.error_handler.call(e, instrumentation_payload)
+          raise e
+        end
+      end
+    end
 
-      processor.configure(consumer: consumer, producer: producer)
+    def process_batch(messages)
+      first, last = messages.first, messages.last
+      instrumentation_payload = {
+        consumer_class: processor.class.to_s,
+        topic:          first.topic,
+        partition:      first.partition,
+        first_offset:   first.offset,
+        last_offset:    last.offset,
+        last_create_time: last.timestamp,
+        message_count:  messages.size
+      }
 
-      begin
-        if processor.respond_to?(:process)
-          consumer.each_message(max_wait_time: config.max_wait_time, max_bytes: config.max_bytes) do |message|
-            payload = {
-              consumer_class: processor.class.to_s,
-              topic: message.topic,
-              partition: message.partition,
-              offset: message.offset,
-            }
-
-            @instrumenter.instrument("process_message.racecar", payload) do
-              processor.process(message)
-              producer.deliver_messages
+      @instrumenter.instrument("start_process_batch", instrumentation_payload)
+      @instrumenter.instrument("process_batch", instrumentation_payload) do
+        with_pause(first.topic, first.partition, first.offset..last.offset) do |pause|
+          begin
+            racecar_messages = messages.map do |message|
+              Racecar::Message.new(message, retries_count: pause.pauses_count)
             end
+            processor.process_batch(racecar_messages)
+            processor.deliver!
+            consumer.store_offset(messages.last)
+          rescue => e
+            instrumentation_payload[:retries_count] = pause.pauses_count
+            config.error_handler.call(e, instrumentation_payload)
+            raise e
           end
-        elsif processor.respond_to?(:process_batch)
-          consumer.each_batch(max_wait_time: config.max_wait_time, max_bytes: config.max_bytes) do |batch|
-            payload = {
-              consumer_class: processor.class.to_s,
-              topic: batch.topic,
-              partition: batch.partition,
-              first_offset: batch.first_offset,
-              message_count: batch.messages.count,
-            }
-
-            @instrumenter.instrument("process_batch.racecar", payload) do
-              processor.process_batch(batch)
-              producer.deliver_messages
-            end
-          end
-        else
-          raise NotImplementedError, "Consumer class must implement process or process_batch method"
-        end
-      rescue Kafka::ProcessingError => e
-        @logger.error "Error processing partition #{e.topic}/#{e.partition} at offset #{e.offset}"
-
-        if config.pause_timeout > 0
-          # Pause fetches from the partition. We'll continue processing the other partitions in the topic.
-          # The partition is automatically resumed after the specified timeout, and will continue where we
-          # left off.
-          @logger.warn "Pausing partition #{e.topic}/#{e.partition} for #{config.pause_timeout} seconds"
-          consumer.pause(
-            e.topic,
-            e.partition,
-            timeout: config.pause_timeout,
-            max_timeout: config.max_pause_timeout,
-            exponential_backoff: config.pause_with_exponential_backoff?,
-          )
-        elsif config.pause_timeout == -1
-          # A pause timeout of -1 means indefinite pausing, which in ruby-kafka is done by passing nil as
-          # the timeout.
-          @logger.warn "Pausing partition #{e.topic}/#{e.partition} indefinitely, or until the process is restarted"
-          consumer.pause(e.topic, e.partition, timeout: nil)
         end
+      end
+    end
 
-        config.error_handler.call(e.cause, {
-          topic: e.topic,
-          partition: e.partition,
-          offset: e.offset,
-        })
+    def with_pause(topic, partition, offsets)
+      pause = pauses[topic][partition]
+      return yield pause if config.pause_timeout == 0
 
-        # Restart the consumer loop.
-        retry
-      rescue Kafka::InvalidSessionTimeout
-        raise ConfigError, "`session_timeout` is set either too high or too low"
-      rescue Kafka::Error => e
-        error = "#{e.class}: #{e.message}\n" + e.backtrace.join("\n")
-        @logger.error "Consumer thread crashed: #{error}"
+      begin
+        yield pause
+        # We've successfully processed a batch from the partition, so we can clear the pause.
+        pauses[topic][partition].reset!
+      rescue => e
+        desc = "#{topic}/#{partition}"
+        logger.error "Failed to process #{desc} at #{offsets}: #{e}"
 
-        config.error_handler.call(e)
+        logger.warn "Pausing partition #{desc} for #{pause.backoff_interval} seconds"
+        consumer.pause(topic, partition, offsets.first)
+        pause.pause!
+      end
+    end
 
-        raise
-      else
-        @logger.info "Gracefully shutting down"
+    def resume_paused_partitions
+      return if config.pause_timeout == 0
+
+      pauses.each do |topic, partitions|
+        partitions.each do |partition, pause|
+          instrumentation_payload = {
+            topic:      topic,
+            partition:  partition,
+            duration:   pause.pause_duration,
+            consumer_class: processor.class.to_s,
+          }
+          @instrumenter.instrument("pause_status", instrumentation_payload)
+
+          if pause.paused? && pause.expired?
+            logger.info "Automatically resuming partition #{topic}/#{partition}, pause timeout expired"
+            consumer.resume(topic, partition)
+            pause.resume!
+            # TODO: # During re-balancing we might have lost the paused partition. Check if partition is still in group before seek. ?
+          end
+        end
       end
     end
   end