racecar 1.2.0 → 2.0.0.alpha1

data/lib/racecar/consumer.rb

@@ -1,11 +1,11 @@
 module Racecar
   class Consumer
-    Subscription = Struct.new(:topic, :start_from_beginning, :max_bytes_per_partition)
+    Subscription = Struct.new(:topic, :start_from_beginning, :max_bytes_per_partition, :additional_config)
 
     class << self
       attr_accessor :max_wait_time
       attr_accessor :group_id
-      attr_accessor :offset_retention_time
+      attr_accessor :producer, :consumer
 
       def subscriptions
         @subscriptions ||= []
@@ -20,29 +20,50 @@ module Racecar
       #   of each partition.
       # @param max_bytes_per_partition [Integer] the maximum number of bytes to fetch from
       #   each partition at a time.
+      # @param additional_config [Hash] Configuration properties for consumer.
+      #   See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       # @return [nil]
-      def subscribes_to(*topics, start_from_beginning: true, max_bytes_per_partition: 1048576)
+      def subscribes_to(*topics, start_from_beginning: true, max_bytes_per_partition: 1048576, additional_config: {})
         topics.each do |topic|
-          subscriptions << Subscription.new(topic, start_from_beginning, max_bytes_per_partition)
+          subscriptions << Subscription.new(topic, start_from_beginning, max_bytes_per_partition, additional_config)
         end
       end
     end
 
-    def configure(consumer:, producer:)
-      @_consumer = consumer
-      @_producer = producer
+    def configure(producer:, consumer:, instrumenter:)
+      @producer = producer
+      @consumer = consumer
+      @instrumenter = instrumenter
     end
 
     def teardown; end
 
+    # Delivers messages that got produced.
+    def deliver!
+      @delivery_handles ||= []
+      @delivery_handles.each(&:wait)
+      @delivery_handles.clear
+    end
+
     protected
 
-    def heartbeat
-      @_consumer.trigger_heartbeat
+    # https://github.com/appsignal/rdkafka-ruby#producing-messages
+    def produce(topic:, payload:, key:, headers: nil)
+      @delivery_handles ||= []
+
+      extra_info = {
+        value:       payload,
+        key:         key,
+        topic:       topic,
+        create_time: Time.now,
+      }
+      @instrumenter.instrument("produce_message.racecar", extra_info) do
+        @delivery_handles << @producer.produce(topic: topic, payload: payload, key: key, headers: headers)
+      end
     end
 
-    def produce(value, **options)
-      @_producer.produce(value, **options)
+    def heartbeat
+      warn "DEPRECATION WARNING: Manual heartbeats are not supported and not needed with librdkafka."
     end
   end
 end

data/lib/racecar/consumer_set.rb

@@ -0,0 +1,177 @@
+module Racecar
+  class ConsumerSet
+    def initialize(config, logger)
+      @config, @logger = config, logger
+      raise ArgumentError, "Subscriptions must not be empty when subscribing" if @config.subscriptions.empty?
+
+      @consumers = []
+      @consumer_id_iterator = (0...@config.subscriptions.size).cycle
+
+      @last_poll_read_nil_message = false
+    end
+
+    def poll(timeout_ms)
+      maybe_select_next_consumer
+      retried ||= false
+      msg = current.poll(timeout_ms)
+    rescue Rdkafka::RdkafkaError => e
+      @logger.error "Error for topic subscription #{current_subscription}: #{e}"
+
+      case e.code
+      when :max_poll_exceeded
+        reset_current_consumer
+        raise if retried
+        retried = true
+        retry
+      else
+        raise
+      end
+    ensure
+      @last_poll_read_nil_message = true if msg.nil?
+    end
+
+    # XXX: messages are not guaranteed to be from the same partition
+    def batch_poll(timeout_ms)
+      @batch_started_at = Time.now
+      @messages = []
+      while collect_messages_for_batch? do
+        msg = poll(timeout_ms)
+        break if msg.nil?
+        @messages << msg
+      end
+      @messages
+    end
+
+    def store_offset(message)
+      current.store_offset(message)
+    end
+
+    def commit
+      each_subscribed do |consumer|
+        commit_rescue_no_offset(consumer)
+      end
+    end
+
+    def close
+      each_subscribed(&:close)
+    end
+
+    def current
+      @consumers[@consumer_id_iterator.peek] ||= begin
+        consumer = Rdkafka::Config.new(rdkafka_config(current_subscription)).consumer
+        consumer.subscribe current_subscription.topic
+        consumer
+      end
+    end
+
+    def each_subscribed
+      if block_given?
+        @consumers.each { |c| yield c }
+      else
+        @consumers.each
+      end
+    end
+
+    def pause(topic, partition, offset)
+      consumer, filtered_tpl = find_consumer_by(topic, partition)
+      if !consumer
+        @logger.info "Attempted to pause #{topic}/#{partition}, but we're not subscribed to it"
+        return
+      end
+
+      consumer.pause(filtered_tpl)
+      fake_msg = OpenStruct.new(topic: topic, partition: partition, offset: offset)
+      consumer.seek(fake_msg)
+    end
+
+    def resume(topic, partition)
+      consumer, filtered_tpl = find_consumer_by(topic, partition)
+      if !consumer
+        @logger.info "Attempted to resume #{topic}/#{partition}, but we're not subscribed to it"
+        return
+      end
+
+      consumer.resume(filtered_tpl)
+    end
+
+    alias :each :each_subscribed
+
+    # Subscribe to all topics eagerly, even if there's still messages elsewhere. Usually
+    # that's not needed and Kafka might rebalance if topics are not polled frequently
+    # enough.
+    def subscribe_all
+      @config.subscriptions.size.times do
+        current
+        select_next_consumer
+      end
+    end
+
+    private
+
+    def find_consumer_by(topic, partition)
+      each do |consumer|
+        tpl = consumer.assignment.to_h
+        rdkafka_partition = tpl[topic]&.detect { |part| part.partition == partition }
+        next unless rdkafka_partition
+        filtered_tpl = Rdkafka::Consumer::TopicPartitionList.new({ topic => [rdkafka_partition] })
+        return consumer, filtered_tpl
+      end
+
+      return nil, nil
+    end
+
+    def current_subscription
+      @config.subscriptions[@consumer_id_iterator.peek]
+    end
+
+    def reset_current_consumer
+      @consumers[@consumer_id_iterator.peek] = nil
+    end
+
+    def maybe_select_next_consumer
+      return unless @last_poll_read_nil_message
+      @last_poll_read_nil_message = false
+      select_next_consumer
+    end
+
+    def select_next_consumer
+      @consumer_id_iterator.next
+    end
+
+    def commit_rescue_no_offset(consumer)
+      consumer.commit(nil, !@config.synchronous_commits)
+    rescue Rdkafka::RdkafkaError => e
+      raise e if e.code != :no_offset
+      @logger.debug "Nothing to commit."
+    end
+
+    def collect_messages_for_batch?
+      @messages.size < @config.fetch_messages &&
+      (Time.now - @batch_started_at) < @config.max_wait_time
+    end
+
+    def rdkafka_config(subscription)
+      # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+      config = {
+        "auto.commit.interval.ms" => @config.offset_commit_interval * 1000,
+        "auto.offset.reset"       => subscription.start_from_beginning ? "earliest" : "largest",
+        "bootstrap.servers"       => @config.brokers.join(","),
+        "client.id"               => @config.client_id,
+        "enable.partition.eof"    => false,
+        "fetch.max.bytes"         => @config.max_bytes,
+        "fetch.message.max.bytes" => subscription.max_bytes_per_partition,
+        "fetch.wait.max.ms"       => @config.max_wait_time * 1000,
+        "group.id"                => @config.group_id,
+        "heartbeat.interval.ms"   => @config.heartbeat_interval * 1000,
+        "max.poll.interval.ms"    => @config.max_poll_interval * 1000,
+        "queued.min.messages"     => @config.min_message_queue_size,
+        "session.timeout.ms"      => @config.session_timeout * 1000,
+        "socket.timeout.ms"       => @config.socket_timeout * 1000,
+        "statistics.interval.ms"  => 1000, # 1s is the highest granularity offered
+      }
+      config.merge! @config.rdkafka_consumer
+      config.merge! subscription.additional_config
+      config
+    end
+  end
+end

data/lib/racecar/ctl.rb

@@ -93,15 +93,12 @@ module Racecar
 
       Racecar.config.validate!
 
-      kafka = Kafka.new(
-        client_id: Racecar.config.client_id,
-        seed_brokers: Racecar.config.brokers,
-        logger: Racecar.logger,
-        connect_timeout: Racecar.config.connect_timeout,
-        socket_timeout: Racecar.config.socket_timeout,
-      )
-
-      kafka.deliver_message(message.value, key: message.key, topic: message.topic)
+      producer = Rdkafka::Config.new({
+        "bootstrap.servers": Racecar.config.brokers.join(","),
+        "client.id":         Racecar.config.client_id,
+      }.merge(Racecar.config.rdkafka_producer)).producer
+
+      producer.produce(payload: message.value, key: message.key, topic: message.topic).wait
 
       $stderr.puts "=> Delivered message to Kafka cluster"
     end

data/lib/racecar/pause.rb

@@ -0,0 +1,55 @@
+module Racecar
+  class Pause
+    def initialize(timeout: nil, max_timeout: nil, exponential_backoff: false)
+      @started_at = nil
+      @pauses = 0
+      @timeout = timeout
+      @max_timeout = max_timeout
+      @exponential_backoff = exponential_backoff
+    end
+
+    def pause!
+      @started_at = Time.now
+      @ends_at = @started_at + backoff_interval unless @timeout.nil?
+      @pauses += 1
+    end
+
+    def resume!
+      @started_at = nil
+      @ends_at = nil
+    end
+
+    def paused?
+      !@started_at.nil?
+    end
+
+    def pause_duration
+      if paused?
+        Time.now - @started_at
+      else
+        0
+      end
+    end
+
+    def expired?
+      return false if @timeout.nil?
+      return true unless @ends_at
+      Time.now >= @ends_at
+    end
+
+    def reset!
+      @pauses = 0
+    end
+
+    def backoff_interval
+      return Float::INFINITY if @timeout.nil?
+
+      backoff_factor = @exponential_backoff ? 2**@pauses : 1
+      timeout = backoff_factor * @timeout
+
+      timeout = @max_timeout if @max_timeout && timeout > @max_timeout
+
+      timeout
+    end
+  end
+end

data/lib/racecar/runner.rb

@@ -1,151 +1,221 @@
-require "kafka"
+require "rdkafka"
+require "racecar/pause"
 
 module Racecar
   class Runner
-    attr_reader :processor, :config, :logger, :consumer
+    attr_reader :processor, :config, :logger
 
     def initialize(processor, config:, logger:, instrumenter: NullInstrumenter)
       @processor, @config, @logger = processor, config, logger
       @instrumenter = instrumenter
+      @stop_requested = false
+      Rdkafka::Config.logger = logger
+
+      if processor.respond_to?(:statistics_callback)
+        Rdkafka::Config.statistics_callback = processor.method(:statistics_callback).to_proc
+      end
+
+      setup_pauses
     end
 
-    def stop
-      Thread.new do
-        processor.teardown
-        consumer.stop unless consumer.nil?
-      end.join
+    def setup_pauses
+      timeout = if config.pause_timeout == -1
+        nil
+      elsif config.pause_timeout == 0
+        # no op, handled elsewhere
+      elsif config.pause_timeout > 0
+        config.pause_timeout
+      else
+        raise ArgumentError, "Invalid value for pause_timeout: must be integer greater or equal -1"
+      end
+
+      @pauses = Hash.new {|h, k|
+        h[k] = Hash.new {|h2, k2|
+          h2[k2] = ::Racecar::Pause.new(
+            timeout:             timeout,
+            max_timeout:         config.max_pause_timeout,
+            exponential_backoff: config.pause_with_exponential_backoff
+          )
+        }
+      }
     end
 
     def run
-      kafka = Kafka.new(
-        client_id: config.client_id,
-        seed_brokers: config.brokers,
-        logger: logger,
-        connect_timeout: config.connect_timeout,
-        socket_timeout: config.socket_timeout,
-        ssl_ca_cert: config.ssl_ca_cert,
-        ssl_ca_cert_file_path: config.ssl_ca_cert_file_path,
-        ssl_client_cert: config.ssl_client_cert,
-        ssl_client_cert_key: config.ssl_client_cert_key,
-        ssl_client_cert_key_password: config.ssl_client_cert_key_password,
-        sasl_plain_username: config.sasl_plain_username,
-        sasl_plain_password: config.sasl_plain_password,
-        sasl_scram_username: config.sasl_scram_username,
-        sasl_scram_password: config.sasl_scram_password,
-        sasl_scram_mechanism: config.sasl_scram_mechanism,
-        sasl_oauth_token_provider: config.sasl_oauth_token_provider,
-        sasl_over_ssl: config.sasl_over_ssl,
-        ssl_ca_certs_from_system: config.ssl_ca_certs_from_system,
-        ssl_verify_hostname: config.ssl_verify_hostname
+      install_signal_handlers
+      @stop_requested = false
+
+      # Configure the consumer with a producer so it can produce messages and
+      # with a consumer so that it can support advanced use-cases.
+      processor.configure(
+        producer:     producer,
+        consumer:     consumer,
+        instrumenter: @instrumenter,
       )
 
-      @consumer = kafka.consumer(
-        group_id: config.group_id,
-        offset_commit_interval: config.offset_commit_interval,
-        offset_commit_threshold: config.offset_commit_threshold,
-        session_timeout: config.session_timeout,
-        heartbeat_interval: config.heartbeat_interval,
-        offset_retention_time: config.offset_retention_time,
-        fetcher_max_queue_size: config.max_fetch_queue_size,
-      )
+      instrument_payload = { consumer_class: processor.class.to_s, consumer_set: consumer }
+
+      # Main loop
+      loop do
+        break if @stop_requested
+        resume_paused_partitions
+        @instrumenter.instrument("main_loop.racecar", instrument_payload) do
+          case process_method
+          when :batch then
+            msg_per_part = consumer.batch_poll(config.max_wait_time).group_by(&:partition)
+            msg_per_part.each_value do |messages|
+              process_batch(messages)
+            end
+          when :single then
+            message = consumer.poll(config.max_wait_time)
+            process(message) if message
+          end
+        end
+      end
+
+      logger.info "Gracefully shutting down"
+      processor.deliver!
+      processor.teardown
+      consumer.commit
+      consumer.close
+    end
 
+    def stop
+      @stop_requested = true
+    end
+
+    private
+
+    attr_reader :pauses
+
+    def process_method
+      @process_method ||= begin
+        case
+        when processor.respond_to?(:process_batch) then :batch
+        when processor.respond_to?(:process) then :single
+        else
+          raise NotImplementedError, "Consumer class must implement process or process_batch method"
+        end
+      end
+    end
+
+    def consumer
+      @consumer ||= begin
+        # Manually store offset after messages have been processed successfully
+        # to avoid marking failed messages as committed. The call just updates
+        # a value within librdkafka and is asynchronously written to proper
+        # storage through auto commits.
+        config.consumer << "enable.auto.offset.store=false"
+        ConsumerSet.new(config, logger)
+      end
+    end
+
+    def producer
+      @producer ||= Rdkafka::Config.new(producer_config).producer.tap do |producer|
+        producer.delivery_callback = delivery_callback
+      end
+    end
+
+    def producer_config
+      # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+      producer_config = {
+        "bootstrap.servers"      => config.brokers.join(","),
+        "client.id"              => config.client_id,
+        "statistics.interval.ms" => 1000,
+      }
+      producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
+      producer_config.merge!(config.rdkafka_producer)
+      producer_config
+    end
+
+    def delivery_callback
+      ->(delivery_report) do
+        data = {offset: delivery_report.offset, partition: delivery_report.partition}
+        @instrumenter.instrument("acknowledged_message.racecar", data)
+      end
+    end
+
+    def install_signal_handlers
       # Stop the consumer on SIGINT, SIGQUIT or SIGTERM.
       trap("QUIT") { stop }
-      trap("INT") { stop }
+      trap("INT")  { stop }
       trap("TERM") { stop }
 
       # Print the consumer config to STDERR on USR1.
       trap("USR1") { $stderr.puts config.inspect }
+    end
 
-      config.subscriptions.each do |subscription|
-        consumer.subscribe(
-          subscription.topic,
-          start_from_beginning: subscription.start_from_beginning,
-          max_bytes_per_partition: subscription.max_bytes_per_partition,
-        )
+    def process(message)
+      payload = {
+        consumer_class: processor.class.to_s,
+        topic: message.topic,
+        partition: message.partition,
+        offset: message.offset,
+      }
+
+      @instrumenter.instrument("process_message.racecar", payload) do
+        with_pause(message.topic, message.partition, message.offset..message.offset) do
+          processor.process(message)
+          processor.deliver!
+          consumer.store_offset(message)
+        end
       end
+    end
 
-      # Configure the consumer with a producer so it can produce messages.
-      producer = kafka.producer(
-        compression_codec: config.producer_compression_codec,
-      )
+    def process_batch(messages)
+      payload = {
+        consumer_class: processor.class.to_s,
+        topic: messages.first.topic,
+        partition: messages.first.partition,
+        first_offset: messages.first.offset,
+        message_count: messages.size,
+      }
+
+      @instrumenter.instrument("process_batch.racecar", payload) do
+        first, last = messages.first, messages.last
+        with_pause(first.topic, first.partition, first.offset..last.offset) do
+          processor.process_batch(messages)
+          processor.deliver!
+          consumer.store_offset(messages.last)
+        end
+      end
+    end
 
-      processor.configure(consumer: consumer, producer: producer)
+    def with_pause(topic, partition, offsets)
+      return yield if config.pause_timeout == 0
 
       begin
-        if processor.respond_to?(:process)
-          consumer.each_message(max_wait_time: config.max_wait_time, max_bytes: config.max_bytes) do |message|
-            payload = {
-              consumer_class: processor.class.to_s,
-              topic: message.topic,
-              partition: message.partition,
-              offset: message.offset,
-            }
-
-            @instrumenter.instrument("process_message.racecar", payload) do
-              processor.process(message)
-              producer.deliver_messages
-            end
-          end
-        elsif processor.respond_to?(:process_batch)
-          consumer.each_batch(max_wait_time: config.max_wait_time, max_bytes: config.max_bytes) do |batch|
-            payload = {
-              consumer_class: processor.class.to_s,
-              topic: batch.topic,
-              partition: batch.partition,
-              first_offset: batch.first_offset,
-              message_count: batch.messages.count,
-            }
-
-            @instrumenter.instrument("process_batch.racecar", payload) do
-              processor.process_batch(batch)
-              producer.deliver_messages
-            end
+        yield
+        # We've successfully processed a batch from the partition, so we can clear the pause.
+        pauses[topic][partition].reset!
+      rescue => e
+        desc = "#{topic}/#{partition}"
+        logger.error "Failed to process #{desc} at #{offsets}: #{e}"
+
+        pause = pauses[topic][partition]
+        logger.warn "Pausing partition #{desc} for #{pause.backoff_interval} seconds"
+        consumer.pause(topic, partition, offsets.first)
+        pause.pause!
+      end
+    end
+
+    def resume_paused_partitions
+      return if config.pause_timeout == 0
+
+      pauses.each do |topic, partitions|
+        partitions.each do |partition, pause|
+          @instrumenter.instrument("pause_status.racecar", {
+            topic: topic,
+            partition: partition,
+            duration: pause.pause_duration,
+          })
+
+          if pause.paused? && pause.expired?
+            logger.info "Automatically resuming partition #{topic}/#{partition}, pause timeout expired"
+            consumer.resume(topic, partition)
+            pause.resume!
+            # TODO: # During re-balancing we might have lost the paused partition. Check if partition is still in group before seek. ?
           end
-        else
-          raise NotImplementedError, "Consumer class must implement process or process_batch method"
         end
-      rescue Kafka::ProcessingError => e
-        @logger.error "Error processing partition #{e.topic}/#{e.partition} at offset #{e.offset}"
-
-        if config.pause_timeout > 0
-          # Pause fetches from the partition. We'll continue processing the other partitions in the topic.
-          # The partition is automatically resumed after the specified timeout, and will continue where we
-          # left off.
-          @logger.warn "Pausing partition #{e.topic}/#{e.partition} for #{config.pause_timeout} seconds"
-          consumer.pause(
-            e.topic,
-            e.partition,
-            timeout: config.pause_timeout,
-            max_timeout: config.max_pause_timeout,
-            exponential_backoff: config.pause_with_exponential_backoff?,
-          )
-        elsif config.pause_timeout == -1
-          # A pause timeout of -1 means indefinite pausing, which in ruby-kafka is done by passing nil as
-          # the timeout.
-          @logger.warn "Pausing partition #{e.topic}/#{e.partition} indefinitely, or until the process is restarted"
-          consumer.pause(e.topic, e.partition, timeout: nil)
-        end
-
-        config.error_handler.call(e.cause, {
-          topic: e.topic,
-          partition: e.partition,
-          offset: e.offset,
-        })
-
-        # Restart the consumer loop.
-        retry
-      rescue Kafka::InvalidSessionTimeout
-        raise ConfigError, "`session_timeout` is set either too high or too low"
-      rescue Kafka::Error => e
-        error = "#{e.class}: #{e.message}\n" + e.backtrace.join("\n")
-        @logger.error "Consumer thread crashed: #{error}"
-
-        config.error_handler.call(e)
-
-        raise
-      else
-        @logger.info "Gracefully shutting down"
       end
     end
   end