racecar 2.0.0.beta3 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb8fe69f83627b7d4171bbb4559fee8b28adf069546b41c62c566ae88e13047f
-  data.tar.gz: d8df951b4b2dd42023d65af45db3e5dca5cff008b7ed16a0a021010f240eba70
+  metadata.gz: 90dddff7c52ef8440ab154be364aacd71f687e7121eaa3456e6b580c3717c294
+  data.tar.gz: '092eb0699d0e60537840cd0114497003ee32e18d611a5b386d430fdfeabd7e03'
 SHA512:
-  metadata.gz: fd6c60e893a61255ec9c5326cb05a087d8883e8bcfa369570b1f60f6e02106992fda445f29378de69a6ab79a6ae96b41ed630591e58ccdad71f965e808c479d6
-  data.tar.gz: e5b6d87edefe41262bc479d712160ebcc95ff3627ffeadd08d103d7eb030721e44e55d82895a2cffb8b5e594be211ea0f03f473eb503b2a3a02e760d5a1524d7
+  metadata.gz: cd4d053961a7f228aeda34bb01e758f9caa6356cf4fb7a3e773c14b9976c19ab30587ce23a010e31b115f30d85999bb38c120e0313b99b43cc744b71f273fb3c
+  data.tar.gz: d0d1865ef2c7adcf94e1bf9c551d7bcdc02a223ee50fbfe4e7293b89629ca7853c55c5cfe144ce879c8aa28c3b8939dcd87fc01c54d441369cfe3c4a186643f3

data/.circleci/config.yml

@@ -0,0 +1,56 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@0.1.2
+
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.6.3-stretch-node
+    executor: ruby/default
+    steps:
+      - checkout
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - ruby/bundle-install
+      - run: bundle exec rspec --exclude-pattern='spec/integration/*_spec.rb'
+  integration-tests:
+    docker:
+      - image: circleci/ruby:2.6.3-stretch-node
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_PORT: 9092
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9093
+          KAFKA_PORT: 9093
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9094
+          KAFKA_PORT: 9094
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+    executor: ruby/default
+    steps:
+      - checkout
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - ruby/bundle-install
+      - run: bundle exec rspec --pattern='spec/integration/*_spec.rb'
+
+workflows:
+  version: 2
+  test:
+    jobs:
+      - build
+      - integration-tests

data/CHANGELOG.md

@@ -1,30 +1,39 @@
 # Changelog
 
+## Unreleased
+
+## racecar v2.1.0
+
+* Bump rdkafka to 0.8.0 (#191)
+
 ## racecar v2.0.0
 
-* Replace `ruby-kafka` with `rdkafka-ruby`
-* Removed config option `sasl_over_ssl`
-* [Racecar::Consumer] Do not pause consuming partitions on exception
-* [Racecar::Consumer] `topic`, `payload` and `key` are mandadory to method `produce`
-* [Racecar::Consumer] `process_batch` retrieves an array of messages instead of batch object
-* [Racecar::Consumer] Remove `offset_retention_time`
-* [Racecar::Consumer] Allow providing `additional_config` for subscriptions
-* [Racecar::Consumer] Provide access to `producer` and `consumer`
-* [Racecar::Consumer] Enforce delivering messages with method `deliver!`
-* [Racecar::Consumer] instead of raising when a partition EOF is reached, the result can be queried through `consumer.last_poll_read_partition_eof?`
-* [Racecar::Config] Remove `offset_retention_time`, `connect_timeout` and `offset_commit_threshold`
-* [Racecar::Config] Pass config to `rdkafka-ruby` via `producer` and `consumer`
-* [Racecar::Config] Replace `max_fetch_queue_size` with `min_message_queue_size`
-* [Racecar::Config] Add `synchronous_commits` to control blocking of `consumer.commit` (default `false`)
-* [Racecar::Config] Add `security_protocol` to control protocol between client and broker
-* [Racecar::Config] SSL configuration via `ssl_ca_location`, `ssl_crl_location`, `ssl_keystore_location` and `ssl_keystore_password`
-* [Racecar::Config] SASL configuration via `sasl_mechanism`, `sasl_kerberos_service_name`, `sasl_kerberos_principal`, `sasl_kerberos_kinit_cmd`, `sasl_kerberos_keytab`, `sasl_kerberos_min_time_before_relogin`, `sasl_username` and `sasl_password`
+* Replace `ruby-kafka` with `rdkafka-ruby` as the low-level library underneath Racecar (#91).
+* Fix `max_wait_time` usage (#179).
+* Removed config option `sasl_over_ssl`.
+* [Racecar::Consumer] Do not pause consuming partitions on exception.
+* [Racecar::Consumer] `topic`, `payload` and `key` are mandadory to method `produce`.
+* [Racecar::Consumer] `process_batch` retrieves an array of messages instead of batch object.
+* [Racecar::Consumer] Remove `offset_retention_time`.
+* [Racecar::Consumer] Allow providing `additional_config` for subscriptions.
+* [Racecar::Consumer] Provide access to `producer` and `consumer`.
+* [Racecar::Consumer] Enforce delivering messages with method `deliver!`.
+* [Racecar::Consumer] instead of raising when a partition EOF is reached, the result can be queried through `consumer.last_poll_read_partition_eof?`.
+* [Racecar::Config] Remove `offset_retention_time`, `connect_timeout` and `offset_commit_threshold`.
+* [Racecar::Config] Pass config to `rdkafka-ruby` via `producer` and `consumer`.
+* [Racecar::Config] Replace `max_fetch_queue_size` with `min_message_queue_size`.
+* [Racecar::Config] Add `synchronous_commits` to control blocking of `consumer.commit` (default `false`).
+* [Racecar::Config] Add `security_protocol` to control protocol between client and broker.
+* [Racecar::Config] SSL configuration via `ssl_ca_location`, `ssl_crl_location`, `ssl_keystore_location` and `ssl_keystore_password`.
+* [Racecar::Config] SASL configuration via `sasl_mechanism`, `sasl_kerberos_service_name`, `sasl_kerberos_principal`, `sasl_kerberos_kinit_cmd`, `sasl_kerberos_keytab`, `sasl_kerberos_min_time_before_relogin`, `sasl_username` and `sasl_password`.
 * [Instrumentation] `produce_message.racecar` sent whenever a produced message is queued. Payload includes `topic`, `key`, `value` and `create_time`.
 * [Instrumentation] `acknowledged_message.racecar` send whenever a produced message was successfully received by Kafka. Payload includes `offset` and `partition`, but no message details.
-* [Instrumentation] `rdkafka-ruby` does not yet provide instrumentation [rdkafka-ruby#54](https://github.com/appsignal/rdkafka-ruby/issues/54)
-* [Instrumentation] if processors define a `statistics_callback`, it will be called once every second for every subscription or producer connection. The first argument will be a Hash, for contents see [librdkafka STATISTICS.md](https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md)
+* [Instrumentation] `rdkafka-ruby` does not yet provide instrumentation [rdkafka-ruby#54](https://github.com/appsignal/rdkafka-ruby/issues/54).
+* [Instrumentation] if processors define a `statistics_callback`, it will be called once every second for every subscription or producer connection. The first argument will be a Hash, for contents see [librdkafka STATISTICS.md](https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md).
 * Add current directory to `$LOAD_PATH` only when `--require` option is used (#117).
-* Remove manual heartbeat support, see [Long-running message processing section in README](README.md#long-running-message-processing)
+* Remove manual heartbeat support, see [Long-running message processing section in README](README.md#long-running-message-processing).
+* Rescue exceptions--then log and pass to `on_error`--at the outermost level of `exe/racecar`, so that exceptions raised outside `Cli.run` are not silently discarded (#186).
+* When exceptions with a `cause` are logged, recursively log the `cause` detail, separated by `--- Caused by: ---\n`.
 
 ## racecar v1.0.0
 

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in racecar.gemspec

data/README.md

@@ -1,5 +1,3 @@
-**IMPORTANT:** The `master` branch is unstable, working towards a v2 release that breaks a lot of stuff. Use the `v1-stable` branch if you want to suggest changes.
-
 # Racecar
 
 Racecar is a friendly and easy-to-approach Kafka consumer framework. It allows you to write small applications that process messages stored in Kafka topics while optionally integrating with your Rails models.
@@ -21,6 +19,7 @@ The framework is based on [rdkafka-ruby](https://github.com/appsignal/rdkafka-ru
     7. [Handling errors](#handling-errors)
     8. [Logging](#logging)
     9. [Operations](#operations)
+    10. [Upgrading from v1 to v2](#upgrading-from-v1-to-v2)
 3. [Development](#development)
 4. [Contributing](#contributing)
 5. [Support and Discussion](#support-and-discussion)
@@ -288,6 +287,9 @@ The memory usage limit is roughly estimated as `max_bytes * min_message_queue_si
 * `ssl_crl_location` – Path to CRL for verifying broker's certificate validity
 * `ssl_keystore_location` – Path to client's keystore (PKCS#12) used for authentication
 * `ssl_keystore_password` – Client's keystore (PKCS#12) password
+* `ssl_certificate_location` – Path to the certificate
+* `ssl_key_location` – Path to client's certificate used for authentication
+* `ssl_key_password` – Client's certificate password
 
 #### SASL encryption, authentication & authorization
 
@@ -464,7 +466,7 @@ Racecar.config.on_error do |exception, info|
 end
 ```
 
-It is highly recommended that you set up an error handler.
+It is highly recommended that you set up an error handler. Please note that the `info` object contains different keys and values depending on whether you are using `process` or `process_batch`. See the `instrumentation_payload` object in the `process` and `process_batch` methods in the `Runner` class for the complete list.
 
 
 ### Logging
@@ -481,6 +483,11 @@ In order to gracefully shut down a Racecar consumer process, send it the `SIGTER
 In order to introspect the configuration of a consumer process, send it the `SIGUSR1` signal. This will make Racecar print its configuration to the standard error file descriptor associated with the consumer process, so you'll need to know where that is written to.
 
 
+### Upgrading from v1 to v2
+
+In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency. In general, you should avoid rolling deploys for consumers groups, so it is likely the case that this will just work for you, but it's a good idea to check first.
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 

data/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '2'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.5.1
+    hostname: zookeeper
+    container_name: zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  broker:
+    image: confluentinc/cp-kafka:5.5.1
+    hostname: broker
+    container_name: broker
+    depends_on:
+      - zookeeper
+    ports:
+      - "29092:29092"
+      - "9092:9092"
+      - "9101:9101"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_JMX_PORT: 9101

data/examples/batch_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class BatchConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/examples/cat_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class CatConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/examples/producing_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ProducingConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/exe/racecar

@@ -3,17 +3,42 @@
 require "racecar"
 require "racecar/cli"
 
-begin
-  Racecar::Cli.main(ARGV)
-rescue SignalException => e
-  # We might receive SIGTERM before our signal handler is installed.
-  if Signal.signame(e.signo) == "TERM"
-    exit(0)
-  else
-    raise
+module Racecar
+  class << self
+    def start(argv)
+      Cli.main(argv)
+    rescue SignalException => e
+      # We might receive SIGTERM before our signal handler is installed.
+      if Signal.signame(e.signo) == "TERM"
+        exit(0)
+      else
+        raise
+      end
+    rescue SystemExit
+      raise
+    rescue Exception => e
+      $stderr.puts "=> Crashed: #{exception_with_causes(e)}\n#{e.backtrace.join("\n")}"
+
+      Racecar.config.error_handler.call(e)
+
+      exit(1)
+    else
+      exit(0)
+    end
+
+    private
+
+    def exception_with_causes(e)
+      result = +"#{e.class}: #{e}"
+      if e.cause
+        result << "\n"
+        result << "--- Caused by: ---\n"
+        result << exception_with_causes(e.cause)
+      end
+      result
+    end
   end
-rescue
-  exit(1)
-else
-  exit(0)
 end
+
+# Start your engines!
+Racecar.start(ARGV)

data/lib/ensure_hash_compact.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # only needed when ruby < 2.4 and not using active support
 
 unless {}.respond_to? :compact

data/lib/generators/racecar/consumer_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module Generators
     class ConsumerGenerator < Rails::Generators::NamedBase

data/lib/generators/racecar/install_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module Generators
     class InstallGenerator < Rails::Generators::Base

data/lib/racecar.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "logger"
 
 require "racecar/instrumenter"
@@ -6,6 +8,7 @@ require "racecar/consumer"
 require "racecar/consumer_set"
 require "racecar/runner"
 require "racecar/config"
+require "racecar/version"
 require "ensure_hash_compact"
 
 module Racecar

data/lib/racecar/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "optparse"
 require "logger"
 require "fileutils"
@@ -6,8 +8,10 @@ require "racecar/daemon"
 
 module Racecar
   class Cli
-    def self.main(args)
-      new(args).run
+    class << self
+      def main(args)
+        new(args).run
+      end
     end
 
     def initialize(args)
@@ -16,10 +20,6 @@ module Racecar
       @consumer_name = args.first or raise Racecar::Error, "no consumer specified"
     end
 
-    def config
-      Racecar.config
-    end
-
     def run
       $stderr.puts "=> Starting Racecar consumer #{consumer_name}..."
 
@@ -61,18 +61,16 @@ module Racecar
       processor = consumer_class.new
 
       Racecar.run(processor)
-    rescue => e
-      $stderr.puts "=> Crashed: #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
-
-      config.error_handler.call(e)
-
-      raise
     end
 
     private
 
     attr_reader :consumer_name
 
+    def config
+      Racecar.config
+    end
+
     def daemonize!
       daemon = Daemon.new(File.expand_path(config.pidfile))
 
@@ -110,13 +108,13 @@ module Racecar
         end
 
         Racecar::Config.variables.each do |variable|
-          opt_name = "--" << variable.name.to_s.gsub("_", "-")
+          opt_name = +"--#{variable.name.to_s.gsub('_', '-')}"
           opt_name << " #{variable.type.upcase}" unless variable.boolean?
 
           desc = variable.description || "N/A"
 
           if variable.default
-            desc << " (default: #{variable.default.inspect})"
+            desc += " (default: #{variable.default.inspect})"
           end
 
           opts.on(opt_name, desc) do |value|

data/lib/racecar/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "king_konf"
 
 module Racecar
@@ -28,6 +30,9 @@ module Racecar
     desc "The maxium number of messages that get consumed within one batch"
     integer :fetch_messages, default: 1000
 
+    desc "Minimum number of bytes the broker responds with"
+    integer :fetch_min_bytes, default: 1
+
     desc "Automatically store offset of last message provided to application"
     boolean :synchronous_commits, default: false
 
@@ -82,6 +87,15 @@ module Racecar
     desc "Client's keystore (PKCS#12) password"
     string :ssl_keystore_password
 
+    desc "Path to the certificate used for authentication"
+    string :ssl_certificate_location
+
+    desc "Path to client's certificate used for authentication"
+    string :ssl_key_location
+
+    desc "Client's certificate password"
+    string :ssl_key_password
+
     desc "SASL mechanism to use for authentication"
     string :sasl_mechanism, allowed_values: %w{GSSAPI PLAIN SCRAM-SHA-256 SCRAM-SHA-512}
 
@@ -144,6 +158,10 @@ module Racecar
 
     attr_accessor :subscriptions, :logger
 
+    def max_wait_time_ms
+      max_wait_time * 1000
+    end
+
     def initialize(env: ENV)
       super(env: env)
       @error_handler = proc {}
@@ -180,8 +198,8 @@ module Racecar
         group_id_prefix,
 
         # MyFunnyConsumer => my-funny-consumer
-        consumer_class.name.gsub(/[a-z][A-Z]/) {|str| str[0] << "-" << str[1] }.downcase,
-      ].compact.join("")
+        consumer_class.name.gsub(/[a-z][A-Z]/) { |str| "#{str[0]}-#{str[1]}" }.downcase,
+      ].compact.join
 
       self.subscriptions = consumer_class.subscriptions
       self.max_wait_time = consumer_class.max_wait_time || self.max_wait_time
@@ -217,6 +235,9 @@ module Racecar
         "ssl.crl.location" => ssl_crl_location,
         "ssl.keystore.location" => ssl_keystore_location,
         "ssl.keystore.password" => ssl_keystore_password,
+        "ssl.certificate.location" => ssl_certificate_location,
+        "ssl.key.location" => ssl_key_location,
+        "ssl.key.password" => ssl_key_password,
         "sasl.mechanism" => sasl_mechanism,
         "sasl.kerberos.service.name" => sasl_kerberos_service_name,
         "sasl.kerberos.principal" => sasl_kerberos_principal,

data/lib/racecar/consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   class Consumer
     Subscription = Struct.new(:topic, :start_from_beginning, :max_bytes_per_partition, :additional_config)
@@ -54,21 +56,29 @@ module Racecar
     protected
 
     # https://github.com/appsignal/rdkafka-ruby#producing-messages
-    def produce(payload, topic:, key:, headers: nil, create_time: nil)
+    def produce(payload, topic:, key: nil, partition_key: nil, headers: nil, create_time: nil)
       @delivery_handles ||= []
       message_size = payload.respond_to?(:bytesize) ? payload.bytesize : 0
       instrumentation_payload = {
-        value:        payload,
-        headers:      headers,
-        key:          key,
-        topic:        topic,
+        value: payload,
+        headers: headers,
+        key: key,
+        partition_key: partition_key,
+        topic: topic,
         message_size: message_size,
-        create_time:  Time.now,
-        buffer_size:  @delivery_handles.size
+        create_time: Time.now,
+        buffer_size: @delivery_handles.size,
       }
 
       @instrumenter.instrument("produce_message", instrumentation_payload) do
-        @delivery_handles << @producer.produce(topic: topic, payload: payload, key: key, timestamp: create_time, headers: headers)
+        @delivery_handles << @producer.produce(
+          topic: topic,
+          payload: payload,
+          key: key,
+          partition_key: partition_key,
+          timestamp: create_time,
+          headers: headers,
+        )
       end
     end
 

data/lib/racecar/consumer_set.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   class ConsumerSet
     MAX_POLL_TRIES = 10
@@ -174,8 +176,9 @@ module Racecar
         "client.id"               => @config.client_id,
         "enable.partition.eof"    => false,
         "fetch.max.bytes"         => @config.max_bytes,
-        "fetch.message.max.bytes" => subscription.max_bytes_per_partition,
-        "fetch.wait.max.ms"       => @config.max_wait_time * 1000,
+        "message.max.bytes"       => subscription.max_bytes_per_partition,
+        "fetch.min.bytes"         => @config.fetch_min_bytes,
+        "fetch.wait.max.ms"       => @config.max_wait_time_ms,
         "group.id"                => @config.group_id,
         "heartbeat.interval.ms"   => @config.heartbeat_interval * 1000,
         "max.poll.interval.ms"    => @config.max_poll_interval * 1000,

data/lib/racecar/ctl.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "optparse"
 require "racecar/rails_config_file_loader"
 require "racecar/daemon"

data/lib/racecar/daemon.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   class Daemon
     attr_reader :pidfile

data/lib/racecar/datadog.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 begin
   require "datadog/statsd"
 rescue LoadError
@@ -26,7 +28,7 @@ module Racecar
       end
 
       def host
-        @host ||= default_host
+        @host
       end
 
       def host=(host)
@@ -35,7 +37,7 @@ module Racecar
       end
 
       def port
-        @port ||= default_port
+        @port
       end
 
       def port=(port)
@@ -63,22 +65,6 @@ module Racecar
 
       private
 
-      def default_host
-        if ::Datadog::Statsd.const_defined?(:Connection)
-          ::Datadog::Statsd::Connection::DEFAULT_HOST
-        else
-          ::Datadog::Statsd::DEFAULT_HOST
-        end
-      end
-
-      def default_port
-        if ::Datadog::Statsd.const_defined?(:Connection)
-          ::Datadog::Statsd::Connection::DEFAULT_PORT
-        else
-          ::Datadog::Statsd::DEFAULT_PORT
-        end
-      end
-
       def clear
         @statsd && @statsd.close
         @statsd = nil
@@ -126,6 +112,8 @@ module Racecar
       def process_batch(event)
         offset = event.payload.fetch(:last_offset)
         messages = event.payload.fetch(:message_count)
+        last_create_time = event.payload.fetch(:last_create_time)
+        time_lag = last_create_time && ((Time.now - last_create_time) * 1000).to_i
         tags = default_tags(event)
 
         if event.payload.key?(:exception)
@@ -135,7 +123,12 @@ module Racecar
           count("consumer.messages", messages, tags: tags)
         end
 
+        histogram("consumer.batch_size", messages, tags: tags)
         gauge("consumer.offset", offset, tags: tags)
+
+        if time_lag
+          gauge("consumer.time_lag", time_lag, tags: tags)
+        end
       end
 
       def join_group(event)

data/lib/racecar/instrumenter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   ##
   # Common API for instrumentation to standardize
@@ -11,6 +13,8 @@ module Racecar
       @default_payload = default_payload
 
       @backend = if defined?(ActiveSupport::Notifications)
+        # ActiveSupport needs `concurrent-ruby` but doesn't `require` it.
+        require 'concurrent/utility/monotonic_time'
         ActiveSupport::Notifications
       else
         NullInstrumenter

data/lib/racecar/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "forwardable"
 
 module Racecar

data/lib/racecar/null_instrumenter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   # Ignores all instrumentation events.
   class NullInstrumenter

data/lib/racecar/pause.rb

@@ -1,8 +1,12 @@
+# frozen_string_literal: true
+
 module Racecar
   class Pause
+    attr_reader :pauses_count
+
     def initialize(timeout: nil, max_timeout: nil, exponential_backoff: false)
       @started_at = nil
-      @pauses = 0
+      @pauses_count = 0
       @timeout = timeout
       @max_timeout = max_timeout
       @exponential_backoff = exponential_backoff
@@ -11,7 +15,7 @@ module Racecar
     def pause!
       @started_at = Time.now
       @ends_at = @started_at + backoff_interval unless @timeout.nil?
-      @pauses += 1
+      @pauses_count += 1
     end
 
     def resume!
@@ -38,13 +42,13 @@ module Racecar
     end
 
     def reset!
-      @pauses = 0
+      @pauses_count = 0
     end
 
     def backoff_interval
       return Float::INFINITY if @timeout.nil?
 
-      backoff_factor = @exponential_backoff ? 2**@pauses : 1
+      backoff_factor = @exponential_backoff ? 2**@pauses_count : 1
       timeout = backoff_factor * @timeout
 
       timeout = @max_timeout if @max_timeout && timeout > @max_timeout

data/lib/racecar/rails_config_file_loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module RailsConfigFileLoader
     def self.load!

data/lib/racecar/runner.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "rdkafka"
 require "racecar/pause"
 require "racecar/message"
@@ -65,12 +67,12 @@ module Racecar
         @instrumenter.instrument("main_loop", instrumentation_payload) do
           case process_method
           when :batch then
-            msg_per_part = consumer.batch_poll(config.max_wait_time).group_by(&:partition)
+            msg_per_part = consumer.batch_poll(config.max_wait_time_ms).group_by(&:partition)
             msg_per_part.each_value do |messages|
               process_batch(messages)
             end
           when :single then
-            message = consumer.poll(config.max_wait_time)
+            message = consumer.poll(config.max_wait_time_ms)
             process(message) if message
           end
         end
@@ -166,11 +168,17 @@ module Racecar
       }
 
       @instrumenter.instrument("start_process_message", instrumentation_payload)
-      @instrumenter.instrument("process_message", instrumentation_payload) do
-        with_pause(message.topic, message.partition, message.offset..message.offset) do
-          processor.process(Racecar::Message.new(message))
-          processor.deliver!
-          consumer.store_offset(message)
+      with_pause(message.topic, message.partition, message.offset..message.offset) do |pause|
+        begin
+          @instrumenter.instrument("process_message", instrumentation_payload) do
+            processor.process(Racecar::Message.new(message))
+            processor.deliver!
+            consumer.store_offset(message)
+          end
+        rescue => e
+          instrumentation_payload[:retries_count] = pause.pauses_count
+          config.error_handler.call(e, instrumentation_payload)
+          raise e
         end
       end
     end
@@ -183,31 +191,38 @@ module Racecar
         partition:      first.partition,
         first_offset:   first.offset,
         last_offset:    last.offset,
+        last_create_time: last.timestamp,
         message_count:  messages.size
       }
 
       @instrumenter.instrument("start_process_batch", instrumentation_payload)
       @instrumenter.instrument("process_batch", instrumentation_payload) do
-        with_pause(first.topic, first.partition, first.offset..last.offset) do
-          processor.process_batch(messages.map {|message| Racecar::Message.new(message) })
-          processor.deliver!
-          consumer.store_offset(messages.last)
+        with_pause(first.topic, first.partition, first.offset..last.offset) do |pause|
+          begin
+            processor.process_batch(messages.map {|message| Racecar::Message.new(message) })
+            processor.deliver!
+            consumer.store_offset(messages.last)
+          rescue => e
+            instrumentation_payload[:retries_count] = pause.pauses_count
+            config.error_handler.call(e, instrumentation_payload)
+            raise e
+          end
         end
       end
     end
 
     def with_pause(topic, partition, offsets)
-      return yield if config.pause_timeout == 0
+      pause = pauses[topic][partition]
+      return yield pause if config.pause_timeout == 0
 
       begin
-        yield
+        yield pause
         # We've successfully processed a batch from the partition, so we can clear the pause.
         pauses[topic][partition].reset!
       rescue => e
         desc = "#{topic}/#{partition}"
         logger.error "Failed to process #{desc} at #{offsets}: #{e}"
 
-        pause = pauses[topic][partition]
         logger.warn "Pausing partition #{desc} for #{pause.backoff_interval} seconds"
         consumer.pause(topic, partition, offsets.first)
         pause.pause!

data/lib/racecar/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
-  VERSION = "2.0.0.beta3"
+  VERSION = "2.1.0"
 end

data/racecar.gemspec

@@ -21,12 +21,13 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "king_konf", "~> 0.3.7"
-  spec.add_runtime_dependency "rdkafka",   "~> 0.6.0"
+  spec.add_runtime_dependency "rdkafka",   "~> 0.8.0"
 
   spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
+  spec.add_development_dependency "pry"
   spec.add_development_dependency "rake", "> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "timecop"
-  spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
+  spec.add_development_dependency "dogstatsd-ruby", ">= 4.0.0", "< 5.0.0"
   spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta3
+  version: 2.1.0
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-02-06 00:00:00.000000000 Z
+date: 2020-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -59,6 +59,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -107,7 +121,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
@@ -117,7 +131,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
@@ -151,6 +165,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".github/workflows/rspec.yml"
 - ".gitignore"
 - ".rspec"
@@ -162,6 +177,7 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- docker-compose.yml
 - examples/batch_consumer.rb
 - examples/cat_consumer.rb
 - examples/producing_consumer.rb
@@ -203,12 +219,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A framework for running Kafka consumers