racecar 2.0.0 → 2.10.0.beta2

data/lib/racecar/message_delivery_error.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Racecar
+  # MessageDeliveryError wraps an Rdkafka error and tries to give
+  # specific hints on how to debug or resolve the error within the
+  # Racecar context.
+  class MessageDeliveryError < StandardError
+    # partition_from_delivery_handle takes an rdkafka delivery handle
+    # and returns a human readable version of the partition. It handles
+    # the case where the partition is unknown.
+    def self.partition_from_delivery_handle(delivery_handle)
+      partition = delivery_handle&.create_result&.partition
+      # -1 is rdkafka-ruby's default value, which gets eventually set by librdkafka
+      return "no yet known" if partition.nil? || partition == -1
+      partition.to_s
+    end
+
+    def initialize(rdkafka_error, delivery_handle)
+      raise rdkafka_error unless rdkafka_error.is_a?(Rdkafka::RdkafkaError)
+
+      @rdkafka_error = rdkafka_error
+      @delivery_handle = delivery_handle
+    end
+
+    attr_reader :rdkafka_error
+
+    def code
+      @rdkafka_error.code
+    end
+
+    def to_s
+      msg = <<~EOM
+        Message delivery finally failed:
+        #{@rdkafka_error.to_s}
+
+        #{explain}
+      EOM
+    end
+
+    private
+
+    def explain
+      case @rdkafka_error.code
+      when :msg_timed_out # -192
+        <<~EOM
+          Could not deliver message within Racecar.config.message_timeout.
+
+          This can happen for various reasons, but most commonly because the connection to the broker is interrupted or there is no leader available. Check the broker's logs or the network for more insight.
+
+          Upstream documentation:
+          https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#error-local-time-out
+        EOM
+
+      when :msg_size_too_large # 10
+        <<~EOM
+          Could not deliver message, since it is bigger than either the broker's or Racecar's maximum message size.
+
+          The broker's config option on the topic is called "max.message.bytes" and the broker wide default is "message.max.bytes". The client's is "message.max.bytes". Take extra care to distinguish this from similarly named properties for receiving/consuming messages (i.e. Racecar.config.max_bytes is NOT related).
+
+          Racecar's limit is currently not configurable and uses librdkafka's default of 1 MB (10³ bytes). As of writing, librdkafka will send at least one message regardless of this limit. It is therefore very likely you're hitting the broker's limit and not Racecar's/librdkafka's.
+
+          Upstream documentation:
+          broker per topic: https://docs.confluent.io/platform/current/installation/configuration/topic-configs.html#topicconfigs_max.message.bytes
+          broker default:   https://docs.confluent.io/platform/current/installation/configuration/broker-configs.html#brokerconfigs_message.max.bytes
+          client:           https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+        EOM
+
+      when :unknown_topic_or_part # 3
+        partition = self.class.partition_from_delivery_handle(@delivery_handle)
+
+        <<~EOM
+          Could not deliver message, since the targeted topic or partition (#{partition}) does not exist.
+
+          Check that there are no typos, or that the broker's "auto.create.topics.enable" is enabled. For freshly created topics with auto create enabled, this may appear in the beginning (race condition on creation and publishing).
+
+          Upstream documentation:
+          broker setting: https://docs.confluent.io/platform/current/installation/configuration/broker-configs.html#brokerconfigs_auto.create.topics.enable
+          client:         https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#topic-metadata-propagation-for-newly-created-topics
+                          https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#topic-auto-creation
+        EOM
+
+      when :record_list_too_large # 18
+        <<~EOM
+          Tried to deliver more messages in a batch than the broker's segment size.
+
+          Either increase the broker's "log.segment.bytes", or decrease any of the client's related settings "batch.num.messages", "batch.size" or "message.max.bytes". None of these are configurable through Racecar yet, as the defaults should be sufficient and sane.
+
+          Upstream documentation:
+          broker: https://docs.confluent.io/platform/current/installation/configuration/broker-configs.html#brokerconfigs_log.segment.bytes
+          client: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+        EOM
+
+      when :topic_authorization_failed # 29
+        <<~EOM
+          Failed to deliver message because of insufficient authorization to write into the topic.
+
+          Double check that it is not a race condition on topic creation. If it isn't, verify the ACLs are correct.
+
+          Upstream documentation:
+          https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#unknown-or-unauthorized-topics
+        EOM
+
+      else
+        <<~EOM
+          No specific information is available for this error. Consider adding it to Racecar. You can find generally helpful information in the upstream documentation:
+          https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md
+          https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+        EOM
+      end
+    end
+  end
+end

data/lib/racecar/null_instrumenter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   # Ignores all instrumentation events.
   class NullInstrumenter

data/lib/racecar/parallel_runner.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Racecar
+  class ParallelRunner
+    Worker = Struct.new(:pid, :parent_reader)
+
+    SHUTDOWN_SIGNALS = ["INT", "QUIT", "TERM"]
+
+    def initialize(runner:, config:, logger:)
+      @runner = runner
+      @config = config
+      @logger = logger
+    end
+
+    def worker_pids
+      workers.map(&:pid)
+    end
+
+    def running?
+      @running
+    end
+
+    def run
+      logger.info "=> Running with #{config.parallel_workers} parallel workers"
+
+      self.workers = config.parallel_workers.times.map do
+        run_worker.tap { |w| logger.info "=> Forked new Racecar consumer with process id #{w.pid}" }
+      end
+
+      # Print the consumer config to STDERR on USR1.
+      trap("USR1") { $stderr.puts config.inspect }
+
+      SHUTDOWN_SIGNALS.each { |signal| trap(signal) { terminate_workers } }
+
+      @running = true
+
+      wait_for_exit
+    end
+
+    def stop
+      terminate_workers
+    end
+
+    private
+
+    attr_accessor :workers
+    attr_reader :runner, :config, :logger
+
+    def run_worker
+      parent_reader, child_writer = IO.pipe
+
+      pid = fork do
+        begin
+          parent_reader.close
+
+          runner.run
+        rescue Exception => e
+          # Allow the parent process to re-raise the exception after shutdown
+          child_writer.binmode
+          child_writer.write(Marshal.dump(e))
+        ensure
+          child_writer.close
+        end
+      end
+
+      child_writer.close
+
+      Worker.new(pid, parent_reader)
+    end
+
+    def terminate_workers
+      return if @terminating
+
+      @terminating = true
+      $stderr.puts "=> Terminating workers"
+
+      Process.kill("TERM", *workers.map(&:pid))
+    end
+
+    def wait_for_exit
+      # The call to IO.select blocks until one or more of our readers are ready for reading,
+      # which could be for one of two reasons:
+      #
+      # - An exception is raised in the child process, in which case we should initiate
+      #   a shutdown;
+      #
+      # - A graceful shutdown was already initiated, and the pipe writer has been closed, in
+      #   which case there is nothing more to do.
+      #
+      # - One of the child processes was killed somehow. If this turns out to be too strict
+      #   (i.e. closing down all the workers, we could revisit and look at restarting dead
+      #   workers.
+      #
+      ready_readers = IO.select(workers.map(&:parent_reader)).first
+
+      first_read = ready_readers.first.read
+
+      terminate_workers
+
+      workers.map(&:pid).each do |pid|
+        logger.debug "=> Waiting for worker with pid #{pid} to exit"
+        Process.waitpid(pid)
+        logger.debug "=> Worker with pid #{pid} shutdown"
+      end
+
+      exception_found = !first_read.empty?
+      raise Marshal.load(first_read) if exception_found
+    end
+  end
+end

data/lib/racecar/pause.rb

@@ -1,8 +1,12 @@
+# frozen_string_literal: true
+
 module Racecar
   class Pause
+    attr_reader :pauses_count
+
     def initialize(timeout: nil, max_timeout: nil, exponential_backoff: false)
       @started_at = nil
-      @pauses = 0
+      @pauses_count = 0
       @timeout = timeout
       @max_timeout = max_timeout
       @exponential_backoff = exponential_backoff
@@ -11,7 +15,7 @@ module Racecar
     def pause!
       @started_at = Time.now
       @ends_at = @started_at + backoff_interval unless @timeout.nil?
-      @pauses += 1
+      @pauses_count += 1
     end
 
     def resume!
@@ -38,13 +42,13 @@ module Racecar
     end
 
     def reset!
-      @pauses = 0
+      @pauses_count = 0
     end
 
     def backoff_interval
       return Float::INFINITY if @timeout.nil?
 
-      backoff_factor = @exponential_backoff ? 2**@pauses : 1
+      backoff_factor = @exponential_backoff ? 2**@pauses_count : 1
       timeout = backoff_factor * @timeout
 
       timeout = @max_timeout if @max_timeout && timeout > @max_timeout

data/lib/racecar/producer.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "racecar/message_delivery_error"
+require "racecar/delivery_callback"
+
+at_exit do
+  Racecar::Producer.shutdown!
+end
+
+module Racecar
+  class Producer
+
+    @@mutex = Mutex.new
+
+    class << self
+      def shutdown!
+        @@mutex.synchronize do
+          if !@internal_producer.nil?
+            @internal_producer.close
+          end
+        end
+      end
+    end
+
+    def initialize(config: nil, logger: nil, instrumenter: NullInstrumenter)
+      @config = config
+      @logger = logger
+      @delivery_handles = []
+      @instrumenter = instrumenter
+      @batching = false
+      @internal_producer = init_internal_producer(config)
+    end
+
+    def init_internal_producer(config)
+      @@mutex.synchronize do
+        @@init_internal_producer ||= begin
+          # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+          producer_config = {
+            "bootstrap.servers"      => config.brokers.join(","),
+            "client.id"              => config.client_id,
+            "statistics.interval.ms" => config.statistics_interval_ms,
+            "message.timeout.ms"     => config.message_timeout * 1000,
+          }
+          producer_config["compression.codec"] = config.producer_compression_codec.to_s unless config.producer_compression_codec.nil?
+          producer_config.merge!(config.rdkafka_producer)
+          Rdkafka::Config.new(producer_config).producer.tap do |producer|
+            producer.delivery_callback = DeliveryCallback.new(instrumenter: @instrumenter)
+          end
+        end
+      end
+    end
+
+    # fire and forget - you won't get any guarantees or feedback from
+    # Racecar on the status of the message and it won't halt execution
+    # of the rest of your code.
+    def produce_async(value:, topic:, **options)
+      with_instrumentation(action: "produce_async", value: value, topic: topic, **options) do
+        begin
+          handle = internal_producer.produce(payload: value, topic: topic, **options)
+          @delivery_handles << handle if @batching
+        rescue Rdkafka::RdkafkaError => e
+          raise MessageDeliveryError.new(e, handle)
+        end
+      end
+
+      nil
+    end
+
+    # synchronous message production - will wait until the delivery handle succeeds, fails or times out.
+    def produce_sync(value:, topic:, **options)
+      with_instrumentation(action: "produce_sync", value: value, topic: topic, **options) do
+        begin
+          handle = internal_producer.produce(payload: value, topic: topic, **options)
+          deliver_with_error_handling(handle)
+        rescue Rdkafka::RdkafkaError => e
+          raise MessageDeliveryError.new(e, handle)
+        end
+      end
+
+      nil
+    end
+
+    # Blocks until all messages that have been asynchronously produced in the block have been delivered.
+    # Usage:
+    # messages = [
+    #             {value: "message1", topic: "topic1"},
+    #             {value: "message2", topic: "topic1"},
+    #             {value: "message3", topic: "topic2"}
+    #             ]
+    # Racecar.wait_for_delivery {
+    #   messages.each do |msg|
+    #     Racecar.produce_async(value: msg[:value], topic: msg[:topic])
+    #   end
+    # }
+    def wait_for_delivery
+      @batching = true
+      @delivery_handles.clear
+      yield
+      @delivery_handles.each do |handle|
+        deliver_with_error_handling(handle)
+      end
+    ensure
+      @delivery_handles.clear
+      @batching = false
+
+      nil
+    end
+
+    private
+
+    attr_reader :internal_producer
+
+    def deliver_with_error_handling(handle)
+      handle.wait
+    rescue Rdkafka::AbstractHandle::WaitTimeoutError => e
+      partition = MessageDeliveryError.partition_from_delivery_handle(handle)
+      @logger.warn "Still trying to deliver message to (partition #{partition})... (will try up to Racecar.config.message_timeout)"
+      retry
+    rescue Rdkafka::RdkafkaError => e
+      raise MessageDeliveryError.new(e, handle)
+    end
+
+    def with_instrumentation(action:, value:, topic:, **options)
+      message_size = value.respond_to?(:bytesize) ? value.bytesize : 0
+      instrumentation_payload = {
+        value: value,
+        topic: topic,
+        message_size: message_size,
+        buffer_size: @delivery_handles.size,
+        key: options.fetch(:key, nil),
+        partition: options.fetch(:partition, nil),
+        partition_key: options.fetch(:partition_key, nil)
+      }
+      @instrumenter.instrument(action, instrumentation_payload) do
+        yield
+      end
+    end
+  end
+end

data/lib/racecar/rails_config_file_loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module RailsConfigFileLoader
     def self.load!
@@ -24,7 +26,11 @@ module Racecar
           console = ActiveSupport::Logger.new($stdout)
           console.formatter = Rails.logger.formatter
           console.level = Rails.logger.level
-          Rails.logger.extend(ActiveSupport::Logger.broadcast(console))
+          if ::Rails::VERSION::STRING < "7.1"
+            Rails.logger.extend(ActiveSupport::Logger.broadcast(console))
+          else
+            Rails.logger = ActiveSupport::BroadcastLogger.new(Rails.logger, console)
+          end
         end
       end
     end

data/lib/racecar/rebalance_listener.rb

@@ -0,0 +1,58 @@
+module Racecar
+  class RebalanceListener
+    def initialize(consumer_class, instrumenter)
+      @consumer_class = consumer_class
+      @instrumenter = instrumenter
+      @rdkafka_consumer = nil
+    end
+
+    attr_writer :rdkafka_consumer
+
+    attr_reader :consumer_class, :instrumenter, :rdkafka_consumer
+    private     :consumer_class, :instrumenter, :rdkafka_consumer
+
+    def on_partitions_assigned(rdkafka_topic_partition_list)
+      event = Event.new(rdkafka_consumer: rdkafka_consumer, rdkafka_topic_partition_list: rdkafka_topic_partition_list)
+
+      instrument("partitions_assigned", partitions: event.partition_numbers) do
+        consumer_class.on_partitions_assigned(event)
+      end
+    end
+
+    def on_partitions_revoked(rdkafka_topic_partition_list)
+      event = Event.new(rdkafka_consumer: rdkafka_consumer, rdkafka_topic_partition_list: rdkafka_topic_partition_list)
+
+      instrument("partitions_revoked", partitions: event.partition_numbers) do
+        consumer_class.on_partitions_revoked(event)
+      end
+    end
+
+    private
+
+    def instrument(event, payload, &block)
+      instrumenter.instrument(event, payload, &block)
+    end
+
+    class Event
+      def initialize(rdkafka_topic_partition_list:, rdkafka_consumer:)
+        @__rdkafka_topic_partition_list = rdkafka_topic_partition_list
+        @__rdkafka_consumer = rdkafka_consumer
+      end
+
+      def topic_name
+        __rdkafka_topic_partition_list.to_h.keys.first
+      end
+
+      def partition_numbers
+        __rdkafka_topic_partition_list.to_h.values.flatten.map(&:partition)
+      end
+
+      def empty?
+        __rdkafka_topic_partition_list.empty?
+      end
+
+      # API private and not guaranteed stable
+      attr_reader :__rdkafka_topic_partition_list, :__rdkafka_consumer
+    end
+  end
+end