racecar 0.5.0.beta2 → 2.2.0

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 

data/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '2'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.5.1
+    hostname: zookeeper
+    container_name: zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  broker:
+    image: confluentinc/cp-kafka:5.5.1
+    hostname: broker
+    container_name: broker
+    depends_on:
+      - zookeeper
+    ports:
+      - "29092:29092"
+      - "9092:9092"
+      - "9101:9101"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_JMX_PORT: 9101

data/examples/batch_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class BatchConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/examples/cat_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class CatConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/examples/producing_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ProducingConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/exe/racecar

@@ -3,19 +3,42 @@
 require "racecar"
 require "racecar/cli"
 
-$LOAD_PATH.unshift(Dir.pwd)
+module Racecar
+  class << self
+    def start(argv)
+      Cli.main(argv)
+    rescue SignalException => e
+      # We might receive SIGTERM before our signal handler is installed.
+      if Signal.signame(e.signo) == "TERM"
+        exit(0)
+      else
+        raise
+      end
+    rescue SystemExit
+      raise
+    rescue Exception => e
+      $stderr.puts "=> Crashed: #{exception_with_causes(e)}\n#{e.backtrace.join("\n")}"
 
-begin
-  Racecar::Cli.main(ARGV)
-rescue SignalException => e
-  # We might receive SIGTERM before our signal handler is installed.
-  if Signal.signame(e.signo) == "TERM"
-    exit(0)
-  else
-    raise
+      Racecar.config.error_handler.call(e)
+
+      exit(1)
+    else
+      exit(0)
+    end
+
+    private
+
+    def exception_with_causes(e)
+      result = +"#{e.class}: #{e}"
+      if e.cause
+        result << "\n"
+        result << "--- Caused by: ---\n"
+        result << exception_with_causes(e.cause)
+      end
+      result
+    end
   end
-rescue
-  exit(1)
-else
-  exit(0)
 end
+
+# Start your engines!
+Racecar.start(ARGV)

data/lib/ensure_hash_compact.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# only needed when ruby < 2.4 and not using active support
+
+unless {}.respond_to? :compact
+  # https://github.com/rails/rails/blob/fc5dd0b85189811062c85520fd70de8389b55aeb/activesupport/lib/active_support/core_ext/hash/compact.rb
+  class Hash
+    def compact
+      select { |_, value| !value.nil? }
+    end
+  end
+end

data/lib/generators/racecar/consumer_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module Generators
     class ConsumerGenerator < Rails::Generators::NamedBase

data/lib/generators/racecar/install_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module Generators
     class InstallGenerator < Rails::Generators::Base

data/lib/racecar.rb

@@ -1,17 +1,17 @@
+# frozen_string_literal: true
+
 require "logger"
 
+require "racecar/instrumenter"
+require "racecar/null_instrumenter"
 require "racecar/consumer"
+require "racecar/consumer_set"
 require "racecar/runner"
 require "racecar/config"
+require "racecar/version"
+require "ensure_hash_compact"
 
 module Racecar
-  # Ignores all instrumentation events.
-  class NullInstrumenter
-    def self.instrument(*)
-      yield if block_given?
-    end
-  end
-
   class Error < StandardError
   end
 
@@ -22,6 +22,10 @@ module Racecar
     @config ||= Config.new
   end
 
+  def self.config=(config)
+    @config = config
+  end
+
   def self.configure
     yield config
   end
@@ -35,13 +39,15 @@ module Racecar
   end
 
   def self.instrumenter
-    require "active_support/notifications"
-
-    ActiveSupport::Notifications
-  rescue LoadError
-    logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
-
-    NullInstrumenter
+    @instrumenter ||= begin
+      default_payload = { client_id: config.client_id, group_id: config.group_id }
+
+      Instrumenter.new(default_payload).tap do |instrumenter|
+        if instrumenter.backend == NullInstrumenter
+          logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
+        end
+      end
+    end
   end
 
   def self.run(processor)

data/lib/racecar/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "optparse"
 require "logger"
 require "fileutils"
@@ -6,8 +8,10 @@ require "racecar/daemon"
 
 module Racecar
   class Cli
-    def self.main(args)
-      new(args).run
+    class << self
+      def main(args)
+        new(args).run
+      end
     end
 
     def initialize(args)
@@ -16,17 +20,13 @@ module Racecar
       @consumer_name = args.first or raise Racecar::Error, "no consumer specified"
     end
 
-    def config
-      Racecar.config
-    end
-
     def run
       $stderr.puts "=> Starting Racecar consumer #{consumer_name}..."
 
-      RailsConfigFileLoader.load!
+      RailsConfigFileLoader.load! unless config.without_rails?
 
       if File.exist?("config/racecar.rb")
-        require "config/racecar"
+        require "./config/racecar"
       end
 
       # Find the consumer class by name.
@@ -61,18 +61,16 @@ module Racecar
       processor = consumer_class.new
 
       Racecar.run(processor)
-    rescue => e
-      $stderr.puts "=> Crashed: #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
-
-      config.error_handler.call(e)
-
-      raise
     end
 
     private
 
     attr_reader :consumer_name
 
+    def config
+      Racecar.config
+    end
+
     def daemonize!
       daemon = Daemon.new(File.expand_path(config.pidfile))
 
@@ -94,10 +92,14 @@ module Racecar
     end
 
     def build_parser
+      load_path_modified = false
+
       OptionParser.new do |opts|
         opts.banner = "Usage: racecar MyConsumer [options]"
 
         opts.on("-r", "--require STRING", "Require a library before starting the consumer") do |lib|
+          $LOAD_PATH.unshift(Dir.pwd) unless load_path_modified
+          load_path_modified = true
           require lib
         end
 
@@ -106,13 +108,13 @@ module Racecar
         end
 
         Racecar::Config.variables.each do |variable|
-          opt_name = "--" << variable.name.to_s.gsub("_", "-")
+          opt_name = +"--#{variable.name.to_s.gsub('_', '-')}"
           opt_name << " #{variable.type.upcase}" unless variable.boolean?
 
           desc = variable.description || "N/A"
 
           if variable.default
-            desc << " (default: #{variable.default.inspect})"
+            desc += " (default: #{variable.default.inspect})"
           end
 
           opts.on(opt_name, desc) do |value|
@@ -140,13 +142,14 @@ module Racecar
     end
 
     def configure_datadog
-      require "kafka/datadog"
+      require_relative './datadog'
 
-      datadog = Kafka::Datadog
-      datadog.host = config.datadog_host unless config.datadog_host.nil?
-      datadog.port = config.datadog_port unless config.datadog_port.nil?
-      datadog.namespace = config.datadog_namespace unless config.datadog_namespace.nil?
-      datadog.tags = config.datadog_tags unless config.datadog_tags.nil?
+      Datadog.configure do |datadog|
+        datadog.host      = config.datadog_host unless config.datadog_host.nil?
+        datadog.port      = config.datadog_port unless config.datadog_port.nil?
+        datadog.namespace = config.datadog_namespace unless config.datadog_namespace.nil?
+        datadog.tags      = config.datadog_tags unless config.datadog_tags.nil?
+      end
     end
   end
 end

data/lib/racecar/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "king_konf"
 
 module Racecar
@@ -13,17 +15,26 @@ module Racecar
     desc "How frequently to commit offset positions"
     float :offset_commit_interval, default: 10
 
-    desc "How many messages to process before forcing a checkpoint"
-    integer :offset_commit_threshold, default: 0
-
     desc "How often to send a heartbeat message to Kafka"
     float :heartbeat_interval, default: 10
 
-    desc "How long committed offsets will be retained."
-    integer :offset_retention_time
+    desc "The minimum number of messages in the local consumer queue"
+    integer :min_message_queue_size, default: 2000
+
+    desc "Kafka consumer configuration options, separated with '=' -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"
+    list :consumer, default: []
+
+    desc "Kafka producer configuration options, separated with '=' -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"
+    list :producer, default: []
+
+    desc "The maxium number of messages that get consumed within one batch"
+    integer :fetch_messages, default: 1000
+
+    desc "Minimum number of bytes the broker responds with"
+    integer :fetch_min_bytes, default: 1
 
-    desc "The maximum number of fetch responses to keep queued before processing"
-    integer :max_fetch_queue_size, default: 10
+    desc "Automatically store offset of last message provided to application"
+    boolean :synchronous_commits, default: false
 
     desc "How long to pause a partition for if the consumer raises an exception while processing a message -- set to -1 to pause indefinitely"
     float :pause_timeout, default: 10
@@ -37,16 +48,16 @@ module Racecar
     desc "The idle timeout after which a consumer is kicked out of the group"
     float :session_timeout, default: 30
 
-    desc "How long to wait when trying to connect to a Kafka broker"
-    float :connect_timeout, default: 10
+    desc "The maximum time between two message fetches before the consumer is kicked out of the group (in seconds)"
+    integer :max_poll_interval, default: 5*60
 
     desc "How long to wait when trying to communicate with a Kafka broker"
     float :socket_timeout, default: 30
 
-    desc "How long to allow the Kafka brokers to wait before returning messages"
+    desc "How long to allow the Kafka brokers to wait before returning messages (in seconds)"
     float :max_wait_time, default: 1
 
-    desc "The maximum size of message sets returned from a single fetch"
+    desc "Maximum amount of data the broker shall return for a Fetch request"
     integer :max_bytes, default: 10485760
 
     desc "A prefix used when generating consumer group names"
@@ -61,44 +72,53 @@ module Racecar
     desc "The log level for the Racecar logs"
     string :log_level, default: "info"
 
-    desc "A valid SSL certificate authority"
-    string :ssl_ca_cert
+    desc "Protocol used to communicate with brokers"
+    symbol :security_protocol, allowed_values: %i{plaintext ssl sasl_plaintext sasl_ssl}
 
-    desc "The path to a valid SSL certificate authority file"
-    string :ssl_ca_cert_file_path
+    desc "File or directory path to CA certificate(s) for verifying the broker's key"
+    string :ssl_ca_location
 
-    desc "A valid SSL client certificate"
-    string :ssl_client_cert
+    desc "Path to CRL for verifying broker's certificate validity"
+    string :ssl_crl_location
 
-    desc "A valid SSL client certificate key"
-    string :ssl_client_cert_key
+    desc "Path to client's keystore (PKCS#12) used for authentication"
+    string :ssl_keystore_location
 
-    desc "Support for using the CA certs installed on your system by default for SSL. More info, see: https://github.com/zendesk/ruby-kafka/pull/521"
-    boolean :ssl_ca_certs_from_system, default: false
+    desc "Client's keystore (PKCS#12) password"
+    string :ssl_keystore_password
 
-    desc "The GSSAPI principal"
-    string :sasl_gssapi_principal
+    desc "Path to the certificate used for authentication"
+    string :ssl_certificate_location
 
-    desc "Optional GSSAPI keytab"
-    string :sasl_gssapi_keytab
+    desc "Path to client's certificate used for authentication"
+    string :ssl_key_location
 
-    desc "The authorization identity to use"
-    string :sasl_plain_authzid
+    desc "Client's certificate password"
+    string :ssl_key_password
 
-    desc "The username used to authenticate"
-    string :sasl_plain_username
+    desc "SASL mechanism to use for authentication"
+    string :sasl_mechanism, allowed_values: %w{GSSAPI PLAIN SCRAM-SHA-256 SCRAM-SHA-512}
 
-    desc "The password used to authenticate"
-    string :sasl_plain_password
+    desc "Kerberos principal name that Kafka runs as, not including /hostname@REALM"
+    string :sasl_kerberos_service_name
 
-    desc "The username used to authenticate"
-    string :sasl_scram_username
+    desc "This client's Kerberos principal name"
+    string :sasl_kerberos_principal
 
-    desc "The password used to authenticate"
-    string :sasl_scram_password
+    desc "Full kerberos kinit command string, %{config.prop.name} is replaced by corresponding config object value, %{broker.name} returns the broker's hostname"
+    string :sasl_kerberos_kinit_cmd
 
-    desc "The SCRAM mechanism to use, either `sha256` or `sha512`"
-    string :sasl_scram_mechanism, allowed_values: ["sha256", "sha512"]
+    desc "Path to Kerberos keytab file. Uses system default if not set"
+    string :sasl_kerberos_keytab
+
+    desc "Minimum time in milliseconds between key refresh attempts"
+    integer :sasl_kerberos_min_time_before_relogin
+
+    desc "SASL username for use with the PLAIN and SASL-SCRAM-.. mechanism"
+    string :sasl_username
+
+    desc "SASL password for use with the PLAIN and SASL-SCRAM-.. mechanism"
+    string :sasl_password
 
     desc "Whether to use SASL over SSL."
     boolean :sasl_over_ssl, default: true
@@ -110,7 +130,7 @@ module Racecar
     boolean :daemonize, default: false
 
     desc "The codec used to compress messages with"
-    symbol :producer_compression_codec
+    symbol :producer_compression_codec, allowed_values: %i{none lz4 snappy gzip}
 
     desc "Enable Datadog metrics"
     boolean :datadog_enabled, default: false
@@ -127,11 +147,21 @@ module Racecar
     desc "Tags that should always be set on Datadog metrics"
     list :datadog_tags
 
+    desc "Whether to check the server certificate is valid for the hostname"
+    boolean :ssl_verify_hostname, default: true
+
+    desc "Whether to boot Rails when starting the consumer"
+    boolean :without_rails, default: false
+
     # The error handler must be set directly on the object.
     attr_reader :error_handler
 
     attr_accessor :subscriptions, :logger
 
+    def max_wait_time_ms
+      max_wait_time * 1000
+    end
+
     def initialize(env: ENV)
       super(env: env)
       @error_handler = proc {}
@@ -155,10 +185,6 @@ module Racecar
         raise ConfigError, "`socket_timeout` must be longer than `max_wait_time`"
       end
 
-      if connect_timeout <= max_wait_time
-        raise ConfigError, "`connect_timeout` must be longer than `max_wait_time`"
-      end
-
       if max_pause_timeout && !pause_with_exponential_backoff?
         raise ConfigError, "`max_pause_timeout` only makes sense when `pause_with_exponential_backoff` is enabled"
       end
@@ -172,17 +198,55 @@ module Racecar
         group_id_prefix,
 
         # MyFunnyConsumer => my-funny-consumer
-        consumer_class.name.gsub(/[a-z][A-Z]/) {|str| str[0] << "-" << str[1] }.downcase,
-      ].compact.join("")
+        consumer_class.name.gsub(/[a-z][A-Z]/) { |str| "#{str[0]}-#{str[1]}" }.downcase,
+      ].compact.join
 
       self.subscriptions = consumer_class.subscriptions
       self.max_wait_time = consumer_class.max_wait_time || self.max_wait_time
-      self.offset_retention_time = consumer_class.offset_retention_time || self.offset_retention_time
       self.pidfile ||= "#{group_id}.pid"
     end
 
     def on_error(&handler)
       @error_handler = handler
     end
+
+    def rdkafka_consumer
+      consumer_config = consumer.map do |param|
+        param.split("=", 2).map(&:strip)
+      end.to_h
+      consumer_config.merge!(rdkafka_security_config)
+      consumer_config
+    end
+
+    def rdkafka_producer
+      producer_config = producer.map do |param|
+        param.split("=", 2).map(&:strip)
+      end.to_h
+      producer_config.merge!(rdkafka_security_config)
+      producer_config
+    end
+
+    private
+
+    def rdkafka_security_config
+      {
+        "security.protocol" => security_protocol,
+        "ssl.ca.location" => ssl_ca_location,
+        "ssl.crl.location" => ssl_crl_location,
+        "ssl.keystore.location" => ssl_keystore_location,
+        "ssl.keystore.password" => ssl_keystore_password,
+        "ssl.certificate.location" => ssl_certificate_location,
+        "ssl.key.location" => ssl_key_location,
+        "ssl.key.password" => ssl_key_password,
+        "sasl.mechanism" => sasl_mechanism,
+        "sasl.kerberos.service.name" => sasl_kerberos_service_name,
+        "sasl.kerberos.principal" => sasl_kerberos_principal,
+        "sasl.kerberos.kinit.cmd" => sasl_kerberos_kinit_cmd,
+        "sasl.kerberos.keytab" => sasl_kerberos_keytab,
+        "sasl.kerberos.min.time.before.relogin" => sasl_kerberos_min_time_before_relogin,
+        "sasl.username" => sasl_username,
+        "sasl.password" => sasl_password,
+      }.compact
+    end
   end
 end