racecar 2.0.0.beta4 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 122d0f341c95b64073cc1ff75891b59c08f9b841ef1245dd2d2a10c70b96677d
-  data.tar.gz: 3e2149507a928c8ecc89c706389d311c7852b4f22aa9d8036a5d0ca44cc4338a
+  metadata.gz: 6380c598cedfca4662aa4c0edf6c55b8a0641d972dad715e3b1e961ac06e4d4f
+  data.tar.gz: 19a30f8515b82f9cbfa783319a76059688ef6096a4dd5e5a5673d6d0c101878c
 SHA512:
-  metadata.gz: f49748fe4e0a549effac3b728dd43edd0885df0a3954ef20daa35f3ac5f994ad582ca0a52ef65fb04dc8dc995f0f7e051c045a2edd5f01617ba3be2fd4d4d939
-  data.tar.gz: f6c118896b4c4a83be5350fbeb2b3ce3be57e18fb68811a39f7aa6dd69b78a174e7ed8bd5536a10210bc37bc979eb7d5c3006160e7c750e60ed9e9ecc767fcd1
+  metadata.gz: 66c6013d77d63a121673e7d4e49fb87b3b1520a3e80a9b564a116b284dbd262ead6fd5b570afa4fa55d77822ac6a6ab31a86917b08630aad2fc9af2f7c08d171
+  data.tar.gz: ce9c89a671478a806dce62979bb695a2673cd598acfb3d2547194a9367a12c96d5f34d38c32f79a3ae809f9eb79303e5330aec4f5e0dc2a23f6f6ffc1e286817

data/.circleci/config.yml

@@ -0,0 +1,56 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@0.1.2
+
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.6.3-stretch-node
+    executor: ruby/default
+    steps:
+      - checkout
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - ruby/bundle-install
+      - run: bundle exec rspec --exclude-pattern='spec/integration/*_spec.rb'
+  integration-tests:
+    docker:
+      - image: circleci/ruby:2.6.3-stretch-node
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_PORT: 9092
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9093
+          KAFKA_PORT: 9093
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9094
+          KAFKA_PORT: 9094
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+    executor: ruby/default
+    steps:
+      - checkout
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - ruby/bundle-install
+      - run: bundle exec rspec --pattern='spec/integration/*_spec.rb'
+
+workflows:
+  version: 2
+  test:
+    jobs:
+      - build
+      - integration-tests

data/CHANGELOG.md

@@ -1,30 +1,44 @@
 # Changelog
 
+## Unreleased
+
+## racecar v2.1.1
+
+* [Bugfix] Close RdKafka consumer in ConsumerSet#reset_current_consumer to prevent memory leak (#196)
+* [Bugfix] `poll`/`batch_poll` would not retry in edge cases and raise immediately. They still honor the `max_wait_time` setting, but might return no messages instead and only retry on their next call. ([#177](https://github.com/zendesk/racecar/pull/177))
+
+## racecar v2.1.0
+
+* Bump rdkafka to 0.8.0 (#191)
+
 ## racecar v2.0.0
 
-* Replace `ruby-kafka` with `rdkafka-ruby`
-* Removed config option `sasl_over_ssl`
-* [Racecar::Consumer] Do not pause consuming partitions on exception
-* [Racecar::Consumer] `topic`, `payload` and `key` are mandadory to method `produce`
-* [Racecar::Consumer] `process_batch` retrieves an array of messages instead of batch object
-* [Racecar::Consumer] Remove `offset_retention_time`
-* [Racecar::Consumer] Allow providing `additional_config` for subscriptions
-* [Racecar::Consumer] Provide access to `producer` and `consumer`
-* [Racecar::Consumer] Enforce delivering messages with method `deliver!`
-* [Racecar::Consumer] instead of raising when a partition EOF is reached, the result can be queried through `consumer.last_poll_read_partition_eof?`
-* [Racecar::Config] Remove `offset_retention_time`, `connect_timeout` and `offset_commit_threshold`
-* [Racecar::Config] Pass config to `rdkafka-ruby` via `producer` and `consumer`
-* [Racecar::Config] Replace `max_fetch_queue_size` with `min_message_queue_size`
-* [Racecar::Config] Add `synchronous_commits` to control blocking of `consumer.commit` (default `false`)
-* [Racecar::Config] Add `security_protocol` to control protocol between client and broker
-* [Racecar::Config] SSL configuration via `ssl_ca_location`, `ssl_crl_location`, `ssl_keystore_location` and `ssl_keystore_password`
-* [Racecar::Config] SASL configuration via `sasl_mechanism`, `sasl_kerberos_service_name`, `sasl_kerberos_principal`, `sasl_kerberos_kinit_cmd`, `sasl_kerberos_keytab`, `sasl_kerberos_min_time_before_relogin`, `sasl_username` and `sasl_password`
+* Replace `ruby-kafka` with `rdkafka-ruby` as the low-level library underneath Racecar (#91).
+* Fix `max_wait_time` usage (#179).
+* Removed config option `sasl_over_ssl`.
+* [Racecar::Consumer] Do not pause consuming partitions on exception.
+* [Racecar::Consumer] `topic`, `payload` and `key` are mandadory to method `produce`.
+* [Racecar::Consumer] `process_batch` retrieves an array of messages instead of batch object.
+* [Racecar::Consumer] Remove `offset_retention_time`.
+* [Racecar::Consumer] Allow providing `additional_config` for subscriptions.
+* [Racecar::Consumer] Provide access to `producer` and `consumer`.
+* [Racecar::Consumer] Enforce delivering messages with method `deliver!`.
+* [Racecar::Consumer] instead of raising when a partition EOF is reached, the result can be queried through `consumer.last_poll_read_partition_eof?`.
+* [Racecar::Config] Remove `offset_retention_time`, `connect_timeout` and `offset_commit_threshold`.
+* [Racecar::Config] Pass config to `rdkafka-ruby` via `producer` and `consumer`.
+* [Racecar::Config] Replace `max_fetch_queue_size` with `min_message_queue_size`.
+* [Racecar::Config] Add `synchronous_commits` to control blocking of `consumer.commit` (default `false`).
+* [Racecar::Config] Add `security_protocol` to control protocol between client and broker.
+* [Racecar::Config] SSL configuration via `ssl_ca_location`, `ssl_crl_location`, `ssl_keystore_location` and `ssl_keystore_password`.
+* [Racecar::Config] SASL configuration via `sasl_mechanism`, `sasl_kerberos_service_name`, `sasl_kerberos_principal`, `sasl_kerberos_kinit_cmd`, `sasl_kerberos_keytab`, `sasl_kerberos_min_time_before_relogin`, `sasl_username` and `sasl_password`.
 * [Instrumentation] `produce_message.racecar` sent whenever a produced message is queued. Payload includes `topic`, `key`, `value` and `create_time`.
 * [Instrumentation] `acknowledged_message.racecar` send whenever a produced message was successfully received by Kafka. Payload includes `offset` and `partition`, but no message details.
-* [Instrumentation] `rdkafka-ruby` does not yet provide instrumentation [rdkafka-ruby#54](https://github.com/appsignal/rdkafka-ruby/issues/54)
-* [Instrumentation] if processors define a `statistics_callback`, it will be called once every second for every subscription or producer connection. The first argument will be a Hash, for contents see [librdkafka STATISTICS.md](https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md)
+* [Instrumentation] `rdkafka-ruby` does not yet provide instrumentation [rdkafka-ruby#54](https://github.com/appsignal/rdkafka-ruby/issues/54).
+* [Instrumentation] if processors define a `statistics_callback`, it will be called once every second for every subscription or producer connection. The first argument will be a Hash, for contents see [librdkafka STATISTICS.md](https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md).
 * Add current directory to `$LOAD_PATH` only when `--require` option is used (#117).
-* Remove manual heartbeat support, see [Long-running message processing section in README](README.md#long-running-message-processing)
+* Remove manual heartbeat support, see [Long-running message processing section in README](README.md#long-running-message-processing).
+* Rescue exceptions--then log and pass to `on_error`--at the outermost level of `exe/racecar`, so that exceptions raised outside `Cli.run` are not silently discarded (#186).
+* When exceptions with a `cause` are logged, recursively log the `cause` detail, separated by `--- Caused by: ---\n`.
 
 ## racecar v1.0.0
 

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in racecar.gemspec

data/README.md

@@ -1,5 +1,3 @@
-**IMPORTANT:** The `master` branch is unstable, working towards a v2 release that breaks a lot of stuff. Use the `v1-stable` branch if you want to suggest changes.
-
 # Racecar
 
 Racecar is a friendly and easy-to-approach Kafka consumer framework. It allows you to write small applications that process messages stored in Kafka topics while optionally integrating with your Rails models.
@@ -21,6 +19,7 @@ The framework is based on [rdkafka-ruby](https://github.com/appsignal/rdkafka-ru
     7. [Handling errors](#handling-errors)
     8. [Logging](#logging)
     9. [Operations](#operations)
+    10. [Upgrading from v1 to v2](#upgrading-from-v1-to-v2)
 3. [Development](#development)
 4. [Contributing](#contributing)
 5. [Support and Discussion](#support-and-discussion)
@@ -51,9 +50,7 @@ This will add a config file in `config/racecar.yml`.
 
 ## Usage
 
-Racecar is built for simplicity of development and operation. If you need more flexibility, it's quite straightforward to build your own Kafka consumer executables using [ruby-kafka](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) directly.
-
-First, a short introduction to the Kafka consumer concept as well as some basic background on Kafka.
+Racecar is built for simplicity of development and operation. First, a short introduction to the Kafka consumer concept as well as some basic background on Kafka.
 
 Kafka stores messages in so-called _partitions_ which are grouped into _topics_. Within a partition, each message gets a unique offset.
 
@@ -227,7 +224,7 @@ You can set message headers by passing a `headers:` option with a Hash of header
 
 Racecar provides a flexible way to configure your consumer in a way that feels at home in a Rails application. If you haven't already, run `bundle exec rails generate racecar:install` in order to generate a config file. You'll get a separate section for each Rails environment, with the common configuration values in a shared `common` section.
 
-**Note:** many of these configuration keys correspond directly to similarly named concepts in [ruby-kafka](https://github.com/zendesk/ruby-kafka); for more details on low-level operations, read that project's documentation.
+**Note:** many of these configuration keys correspond directly to similarly named concepts in [rdkafka-ruby](https://github.com/appsignal/rdkafka-ruby); for more details on low-level operations, read that project's documentation.
 
 It's also possible to configure Racecar using environment variables. For any given configuration key, there should be a corresponding environment variable with the prefix `RACECAR_`, in upper case. For instance, in order to configure the client id, set `RACECAR_CLIENT_ID=some-id` in the process in which the Racecar consumer is launched. You can set `brokers` by passing a comma-separated list, e.g. `RACECAR_BROKERS=kafka1:9092,kafka2:9092,kafka3:9092`.
 
@@ -274,7 +271,7 @@ All timeouts are defined in number of seconds.
 
 Kafka is _really_ good at throwing data at consumers, so you may want to tune these variables in order to avoid ballooning your process' memory or saturating your network capacity.
 
-Racecar uses ruby-kafka under the hood, which fetches messages from the Kafka brokers in a background thread. This thread pushes fetch responses, possible containing messages from many partitions, into a queue that is read by the processing thread (AKA your code). The main way to control the fetcher thread is to control the size of those responses and the size of the queue.
+Racecar uses [rdkafka-ruby](https://github.com/appsignal/rdkafka-ruby) under the hood, which fetches messages from the Kafka brokers in a background thread. This thread pushes fetch responses, possible containing messages from many partitions, into a queue that is read by the processing thread (AKA your code). The main way to control the fetcher thread is to control the size of those responses and the size of the queue.
 
 * `max_bytes` — Maximum amount of data the broker shall return for a Fetch request.
 * `min_message_queue_size` — The minimum number of messages in the local consumer queue.
@@ -314,7 +311,7 @@ These settings are related to consumers that _produce messages to Kafka_.
 
 #### Datadog monitoring
 
-Racecar supports configuring ruby-kafka's [Datadog](https://www.datadoghq.com/) monitoring integration. If you're running a normal Datadog agent on your host, you just need to set `datadog_enabled` to `true`, as the rest of the settings come with sane defaults.
+Racecar supports [Datadog](https://www.datadoghq.com/) monitoring integration. If you're running a normal Datadog agent on your host, you just need to set `datadog_enabled` to `true`, as the rest of the settings come with sane defaults.
 
 * `datadog_enabled` – Whether Datadog monitoring is enabled (defaults to `false`).
 * `datadog_host` – The host running the Datadog agent.
@@ -467,7 +464,7 @@ Racecar.config.on_error do |exception, info|
 end
 ```
 
-It is highly recommended that you set up an error handler.
+It is highly recommended that you set up an error handler. Please note that the `info` object contains different keys and values depending on whether you are using `process` or `process_batch`. See the `instrumentation_payload` object in the `process` and `process_batch` methods in the `Runner` class for the complete list.
 
 
 ### Logging
@@ -484,10 +481,17 @@ In order to gracefully shut down a Racecar consumer process, send it the `SIGTER
 In order to introspect the configuration of a consumer process, send it the `SIGUSR1` signal. This will make Racecar print its configuration to the standard error file descriptor associated with the consumer process, so you'll need to know where that is written to.
 
 
+### Upgrading from v1 to v2
+
+In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency. In general, you should avoid rolling deploys for consumers groups, so it is likely the case that this will just work for you, but it's a good idea to check first.
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
+The integration tests run against a Kafka instance that is not automatically started from within `rspec`. You can set one up using the provided `docker-compose.yml` by running `docker-compose up`.
+
 
 ## Contributing
 
@@ -496,9 +500,9 @@ Bug reports and pull requests are welcome on [GitHub](https://github.com/zendesk
 
 ## Support and Discussion
 
-If you've discovered a bug, please file a [Github issue](https://github.com/zendesk/racecar/issues/new), and make sure to include all the relevant information, including the version of Racecar, ruby-kafka, and Kafka that you're using.
+If you've discovered a bug, please file a [Github issue](https://github.com/zendesk/racecar/issues/new), and make sure to include all the relevant information, including the version of Racecar, rdkafka-ruby, and Kafka that you're using.
 
-If you have other questions, or would like to discuss best practises, how to contribute to the project, or any other ruby-kafka related topic, [join our Slack team](https://ruby-kafka-slack.herokuapp.com/)!
+If you have other questions, or would like to discuss best practises, or how to contribute to the project, [join our Slack team](https://ruby-kafka-slack.herokuapp.com/)!
 
 
 ## Copyright and license

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 

data/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '2'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.5.1
+    hostname: zookeeper
+    container_name: zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  broker:
+    image: confluentinc/cp-kafka:5.5.1
+    hostname: broker
+    container_name: broker
+    depends_on:
+      - zookeeper
+    ports:
+      - "29092:29092"
+      - "9092:9092"
+      - "9101:9101"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_JMX_PORT: 9101

data/examples/batch_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class BatchConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/examples/cat_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class CatConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/examples/producing_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ProducingConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/exe/racecar

@@ -3,17 +3,42 @@
 require "racecar"
 require "racecar/cli"
 
-begin
-  Racecar::Cli.main(ARGV)
-rescue SignalException => e
-  # We might receive SIGTERM before our signal handler is installed.
-  if Signal.signame(e.signo) == "TERM"
-    exit(0)
-  else
-    raise
+module Racecar
+  class << self
+    def start(argv)
+      Cli.main(argv)
+    rescue SignalException => e
+      # We might receive SIGTERM before our signal handler is installed.
+      if Signal.signame(e.signo) == "TERM"
+        exit(0)
+      else
+        raise
+      end
+    rescue SystemExit
+      raise
+    rescue Exception => e
+      $stderr.puts "=> Crashed: #{exception_with_causes(e)}\n#{e.backtrace.join("\n")}"
+
+      Racecar.config.error_handler.call(e)
+
+      exit(1)
+    else
+      exit(0)
+    end
+
+    private
+
+    def exception_with_causes(e)
+      result = +"#{e.class}: #{e}"
+      if e.cause
+        result << "\n"
+        result << "--- Caused by: ---\n"
+        result << exception_with_causes(e.cause)
+      end
+      result
+    end
   end
-rescue
-  exit(1)
-else
-  exit(0)
 end
+
+# Start your engines!
+Racecar.start(ARGV)

data/lib/ensure_hash_compact.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # only needed when ruby < 2.4 and not using active support
 
 unless {}.respond_to? :compact

data/lib/generators/racecar/consumer_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module Generators
     class ConsumerGenerator < Rails::Generators::NamedBase

data/lib/generators/racecar/install_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module Generators
     class InstallGenerator < Rails::Generators::Base

data/lib/racecar.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "logger"
 
 require "racecar/instrumenter"
@@ -6,6 +8,7 @@ require "racecar/consumer"
 require "racecar/consumer_set"
 require "racecar/runner"
 require "racecar/config"
+require "racecar/version"
 require "ensure_hash_compact"
 
 module Racecar

data/lib/racecar/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "optparse"
 require "logger"
 require "fileutils"
@@ -6,8 +8,10 @@ require "racecar/daemon"
 
 module Racecar
   class Cli
-    def self.main(args)
-      new(args).run
+    class << self
+      def main(args)
+        new(args).run
+      end
     end
 
     def initialize(args)
@@ -16,10 +20,6 @@ module Racecar
       @consumer_name = args.first or raise Racecar::Error, "no consumer specified"
     end
 
-    def config
-      Racecar.config
-    end
-
     def run
       $stderr.puts "=> Starting Racecar consumer #{consumer_name}..."
 
@@ -61,18 +61,16 @@ module Racecar
       processor = consumer_class.new
 
       Racecar.run(processor)
-    rescue => e
-      $stderr.puts "=> Crashed: #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
-
-      config.error_handler.call(e)
-
-      raise
     end
 
     private
 
     attr_reader :consumer_name
 
+    def config
+      Racecar.config
+    end
+
     def daemonize!
       daemon = Daemon.new(File.expand_path(config.pidfile))
 
@@ -102,12 +100,7 @@ module Racecar
         opts.on("-r", "--require STRING", "Require a library before starting the consumer") do |lib|
           $LOAD_PATH.unshift(Dir.pwd) unless load_path_modified
           load_path_modified = true
-          begin
-            require lib
-          rescue => e
-            $stderr.puts "=> #{lib} failed to load: #{e.message}"
-            exit
-          end
+          require lib
         end
 
         opts.on("-l", "--log STRING", "Log to the specified file") do |logfile|
@@ -115,13 +108,13 @@ module Racecar
         end
 
         Racecar::Config.variables.each do |variable|
-          opt_name = "--" << variable.name.to_s.gsub("_", "-")
+          opt_name = +"--#{variable.name.to_s.gsub('_', '-')}"
           opt_name << " #{variable.type.upcase}" unless variable.boolean?
 
           desc = variable.description || "N/A"
 
           if variable.default
-            desc << " (default: #{variable.default.inspect})"
+            desc += " (default: #{variable.default.inspect})"
           end
 
           opts.on(opt_name, desc) do |value|

data/lib/racecar/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "king_konf"
 
 module Racecar
@@ -28,6 +30,9 @@ module Racecar
     desc "The maxium number of messages that get consumed within one batch"
     integer :fetch_messages, default: 1000
 
+    desc "Minimum number of bytes the broker responds with"
+    integer :fetch_min_bytes, default: 1
+
     desc "Automatically store offset of last message provided to application"
     boolean :synchronous_commits, default: false
 
@@ -153,6 +158,10 @@ module Racecar
 
     attr_accessor :subscriptions, :logger
 
+    def max_wait_time_ms
+      max_wait_time * 1000
+    end
+
     def initialize(env: ENV)
       super(env: env)
       @error_handler = proc {}
@@ -189,8 +198,8 @@ module Racecar
         group_id_prefix,
 
         # MyFunnyConsumer => my-funny-consumer
-        consumer_class.name.gsub(/[a-z][A-Z]/) {|str| str[0] << "-" << str[1] }.downcase,
-      ].compact.join("")
+        consumer_class.name.gsub(/[a-z][A-Z]/) { |str| "#{str[0]}-#{str[1]}" }.downcase,
+      ].compact.join
 
       self.subscriptions = consumer_class.subscriptions
       self.max_wait_time = consumer_class.max_wait_time || self.max_wait_time

data/lib/racecar/consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   class Consumer
     Subscription = Struct.new(:topic, :start_from_beginning, :max_bytes_per_partition, :additional_config)
@@ -54,21 +56,29 @@ module Racecar
     protected
 
     # https://github.com/appsignal/rdkafka-ruby#producing-messages
-    def produce(payload, topic:, key:, headers: nil, create_time: nil)
+    def produce(payload, topic:, key: nil, partition_key: nil, headers: nil, create_time: nil)
       @delivery_handles ||= []
       message_size = payload.respond_to?(:bytesize) ? payload.bytesize : 0
       instrumentation_payload = {
-        value:        payload,
-        headers:      headers,
-        key:          key,
-        topic:        topic,
+        value: payload,
+        headers: headers,
+        key: key,
+        partition_key: partition_key,
+        topic: topic,
         message_size: message_size,
-        create_time:  Time.now,
-        buffer_size:  @delivery_handles.size
+        create_time: Time.now,
+        buffer_size: @delivery_handles.size,
       }
 
       @instrumenter.instrument("produce_message", instrumentation_payload) do
-        @delivery_handles << @producer.produce(topic: topic, payload: payload, key: key, timestamp: create_time, headers: headers)
+        @delivery_handles << @producer.produce(
+          topic: topic,
+          payload: payload,
+          key: key,
+          partition_key: partition_key,
+          timestamp: create_time,
+          headers: headers,
+        )
       end
     end
 

data/lib/racecar/consumer_set.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   class ConsumerSet
     MAX_POLL_TRIES = 10
@@ -10,49 +12,39 @@ module Racecar
       @consumers = []
       @consumer_id_iterator = (0...@config.subscriptions.size).cycle
 
+      @previous_retries = 0
+
       @last_poll_read_nil_message = false
     end
 
-    def poll(timeout_ms)
-      maybe_select_next_consumer
-      started_at ||= Time.now
-      try ||= 0
-      remain ||= timeout_ms
-
-      msg = remain <= 0 ? nil : current.poll(remain)
-    rescue Rdkafka::RdkafkaError => e
-      wait_before_retry_ms = 100 * (2**try) # 100ms, 200ms, 400ms, …
-      try += 1
-      raise if try >= MAX_POLL_TRIES || remain <= wait_before_retry_ms
-
-      @logger.error "(try #{try}): Error for topic subscription #{current_subscription}: #{e}"
-
-      case e.code
-      when :max_poll_exceeded, :transport # -147, -195
-        reset_current_consumer
-      end
-
-      remain = remaining_time_ms(timeout_ms, started_at)
-      raise if remain <= wait_before_retry_ms
-
-      sleep wait_before_retry_ms/1000.0
-      retry
-    ensure
-      @last_poll_read_nil_message = true if msg.nil?
+    def poll(max_wait_time_ms = @config.max_wait_time)
+      batch_poll(max_wait_time_ms, 1).first
     end
 
-    # XXX: messages are not guaranteed to be from the same partition
-    def batch_poll(timeout_ms)
-      @batch_started_at = Time.now
-      @messages = []
-      while collect_messages_for_batch? do
-        remain = remaining_time_ms(timeout_ms, @batch_started_at)
-        break if remain <= 0
-        msg = poll(remain)
+    # batch_poll collects messages until any of the following occurs:
+    # - max_wait_time_ms time has passed
+    # - max_messages have been collected
+    # - a nil message was polled (end of topic, Kafka stalled, etc.)
+    #
+    # The messages are from a single topic, but potentially from more than one partition.
+    #
+    # Any errors during polling are retried in an exponential backoff fashion. If an error
+    # occurs, but there is no time left for a backoff and retry, it will return the
+    # already collected messages and only retry on the next call.
+    def batch_poll(max_wait_time_ms = @config.max_wait_time, max_messages = @config.fetch_messages)
+      started_at = Time.now
+      remain_ms = max_wait_time_ms
+      maybe_select_next_consumer
+      messages = []
+
+      while remain_ms > 0 && messages.size < max_messages
+        remain_ms = remaining_time_ms(max_wait_time_ms, started_at)
+        msg = poll_with_retries(remain_ms)
         break if msg.nil?
-        @messages << msg
+        messages << msg
       end
-      @messages
+
+      messages
     end
 
     def store_offset(message)
@@ -123,6 +115,52 @@ module Racecar
 
     private
 
+    # polls a single message from the current consumer, retrying errors with exponential
+    # backoff. The sleep time is capped by max_wait_time_ms. If there's enough time budget
+    # left, it will retry before returning. If there isn't, the retry will only occur on
+    # the next call. It tries up to MAX_POLL_TRIES before passing on the exception.
+    def poll_with_retries(max_wait_time_ms)
+      try ||= @previous_retries
+      @previous_retries = 0
+      started_at ||= Time.now
+      remain_ms = remaining_time_ms(max_wait_time_ms, started_at)
+
+      wait_ms = try == 0 ? 0 : 50 * (2**try) # 0ms, 100ms, 200ms, 400ms, …
+      if wait_ms >= max_wait_time_ms && remain_ms > 1
+        @logger.debug "Capping #{wait_ms}ms to #{max_wait_time_ms-1}ms."
+        sleep (max_wait_time_ms-1)/1000.0
+        remain_ms = 1
+      elsif wait_ms >= remain_ms
+        @logger.error "Only #{remain_ms}ms left, but want to wait for #{wait_ms}ms before poll. Will retry on next call."
+        @previous_retries = try
+        return nil
+      elsif wait_ms > 0
+        sleep wait_ms/1000.0
+        remain_ms -= wait_ms
+      end
+
+      poll_current_consumer(remain_ms)
+    rescue Rdkafka::RdkafkaError => e
+      try += 1
+      @instrumenter.instrument("poll_retry", try: try, rdkafka_time_limit: remain_ms, exception: e)
+      @logger.error "(try #{try}/#{MAX_POLL_TRIES}): Error for topic subscription #{current_subscription}: #{e}"
+      raise if try >= MAX_POLL_TRIES
+      retry
+    end
+
+    # polls a message for the current consumer, handling any API edge cases.
+    def poll_current_consumer(max_wait_time_ms)
+      msg = current.poll(max_wait_time_ms)
+    rescue Rdkafka::RdkafkaError => e
+      case e.code
+      when :max_poll_exceeded, :transport # -147, -195
+        reset_current_consumer
+      end
+      raise
+    ensure
+      @last_poll_read_nil_message = msg.nil?
+    end
+
     def find_consumer_by(topic, partition)
       each do |consumer|
         tpl = consumer.assignment.to_h
@@ -140,7 +178,12 @@ module Racecar
     end
 
     def reset_current_consumer
-      @consumers[@consumer_id_iterator.peek] = nil
+      current_consumer_id = @consumer_id_iterator.peek
+      @logger.info "Resetting consumer with id: #{current_consumer_id}"
+
+      consumer = @consumers[current_consumer_id]
+      consumer.close unless consumer.nil?
+      @consumers[current_consumer_id] = nil
     end
 
     def maybe_select_next_consumer
@@ -160,11 +203,6 @@ module Racecar
       @logger.debug "Nothing to commit."
     end
 
-    def collect_messages_for_batch?
-      @messages.size < @config.fetch_messages &&
-      (Time.now - @batch_started_at) < @config.max_wait_time
-    end
-
     def rdkafka_config(subscription)
       # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       config = {
@@ -174,8 +212,9 @@ module Racecar
         "client.id"               => @config.client_id,
         "enable.partition.eof"    => false,
         "fetch.max.bytes"         => @config.max_bytes,
-        "fetch.message.max.bytes" => subscription.max_bytes_per_partition,
-        "fetch.wait.max.ms"       => @config.max_wait_time * 1000,
+        "message.max.bytes"       => subscription.max_bytes_per_partition,
+        "fetch.min.bytes"         => @config.fetch_min_bytes,
+        "fetch.wait.max.ms"       => @config.max_wait_time_ms,
         "group.id"                => @config.group_id,
         "heartbeat.interval.ms"   => @config.heartbeat_interval * 1000,
         "max.poll.interval.ms"    => @config.max_poll_interval * 1000,

data/lib/racecar/ctl.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "optparse"
 require "racecar/rails_config_file_loader"
 require "racecar/daemon"
@@ -98,7 +100,8 @@ module Racecar
         "client.id":         Racecar.config.client_id,
       }.merge(Racecar.config.rdkafka_producer)).producer
 
-      producer.produce(payload: message.value, key: message.key, topic: message.topic).wait
+      handle = producer.produce(payload: message.value, key: message.key, topic: message.topic)
+      handle.wait(max_wait_timeout: 5)
 
       $stderr.puts "=> Delivered message to Kafka cluster"
     end

data/lib/racecar/daemon.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   class Daemon
     attr_reader :pidfile

data/lib/racecar/datadog.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 begin
   require "datadog/statsd"
 rescue LoadError
@@ -155,6 +157,15 @@ module Racecar
         end
       end
 
+      def poll_retry(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+        }
+        rdkafka_error_code = event.payload.fetch(:exception).code.to_s.gsub(/\W/, '')
+        increment("consumer.poll.rdkafka_error.#{rdkafka_error_code}", tags: tags)
+      end
+
       def main_loop(event)
         tags = {
           client: event.payload.fetch(:client_id),

data/lib/racecar/instrumenter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   ##
   # Common API for instrumentation to standardize
@@ -11,6 +13,8 @@ module Racecar
       @default_payload = default_payload
 
       @backend = if defined?(ActiveSupport::Notifications)
+        # ActiveSupport needs `concurrent-ruby` but doesn't `require` it.
+        require 'concurrent/utility/monotonic_time'
         ActiveSupport::Notifications
       else
         NullInstrumenter

data/lib/racecar/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "forwardable"
 
 module Racecar

data/lib/racecar/null_instrumenter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   # Ignores all instrumentation events.
   class NullInstrumenter

data/lib/racecar/pause.rb

@@ -1,8 +1,12 @@
+# frozen_string_literal: true
+
 module Racecar
   class Pause
+    attr_reader :pauses_count
+
     def initialize(timeout: nil, max_timeout: nil, exponential_backoff: false)
       @started_at = nil
-      @pauses = 0
+      @pauses_count = 0
       @timeout = timeout
       @max_timeout = max_timeout
       @exponential_backoff = exponential_backoff
@@ -11,7 +15,7 @@ module Racecar
     def pause!
       @started_at = Time.now
       @ends_at = @started_at + backoff_interval unless @timeout.nil?
-      @pauses += 1
+      @pauses_count += 1
     end
 
     def resume!
@@ -38,13 +42,13 @@ module Racecar
     end
 
     def reset!
-      @pauses = 0
+      @pauses_count = 0
     end
 
     def backoff_interval
       return Float::INFINITY if @timeout.nil?
 
-      backoff_factor = @exponential_backoff ? 2**@pauses : 1
+      backoff_factor = @exponential_backoff ? 2**@pauses_count : 1
       timeout = backoff_factor * @timeout
 
       timeout = @max_timeout if @max_timeout && timeout > @max_timeout

data/lib/racecar/rails_config_file_loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module RailsConfigFileLoader
     def self.load!

data/lib/racecar/runner.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "rdkafka"
 require "racecar/pause"
 require "racecar/message"
@@ -65,12 +67,12 @@ module Racecar
         @instrumenter.instrument("main_loop", instrumentation_payload) do
           case process_method
           when :batch then
-            msg_per_part = consumer.batch_poll(config.max_wait_time).group_by(&:partition)
+            msg_per_part = consumer.batch_poll(config.max_wait_time_ms).group_by(&:partition)
             msg_per_part.each_value do |messages|
               process_batch(messages)
             end
           when :single then
-            message = consumer.poll(config.max_wait_time)
+            message = consumer.poll(config.max_wait_time_ms)
             process(message) if message
           end
         end
@@ -166,7 +168,7 @@ module Racecar
       }
 
       @instrumenter.instrument("start_process_message", instrumentation_payload)
-      with_pause(message.topic, message.partition, message.offset..message.offset) do
+      with_pause(message.topic, message.partition, message.offset..message.offset) do |pause|
         begin
           @instrumenter.instrument("process_message", instrumentation_payload) do
             processor.process(Racecar::Message.new(message))
@@ -174,6 +176,7 @@ module Racecar
             consumer.store_offset(message)
           end
         rescue => e
+          instrumentation_payload[:retries_count] = pause.pauses_count
           config.error_handler.call(e, instrumentation_payload)
           raise e
         end
@@ -194,12 +197,13 @@ module Racecar
 
       @instrumenter.instrument("start_process_batch", instrumentation_payload)
       @instrumenter.instrument("process_batch", instrumentation_payload) do
-        with_pause(first.topic, first.partition, first.offset..last.offset) do
+        with_pause(first.topic, first.partition, first.offset..last.offset) do |pause|
           begin
             processor.process_batch(messages.map {|message| Racecar::Message.new(message) })
             processor.deliver!
             consumer.store_offset(messages.last)
           rescue => e
+            instrumentation_payload[:retries_count] = pause.pauses_count
             config.error_handler.call(e, instrumentation_payload)
             raise e
           end
@@ -208,17 +212,17 @@ module Racecar
     end
 
     def with_pause(topic, partition, offsets)
-      return yield if config.pause_timeout == 0
+      pause = pauses[topic][partition]
+      return yield pause if config.pause_timeout == 0
 
       begin
-        yield
+        yield pause
         # We've successfully processed a batch from the partition, so we can clear the pause.
         pauses[topic][partition].reset!
       rescue => e
         desc = "#{topic}/#{partition}"
         logger.error "Failed to process #{desc} at #{offsets}: #{e}"
 
-        pause = pauses[topic][partition]
         logger.warn "Pausing partition #{desc} for #{pause.backoff_interval} seconds"
         consumer.pause(topic, partition, offsets.first)
         pause.pause!

data/lib/racecar/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
-  VERSION = "2.0.0.beta4"
+  VERSION = "2.1.1"
 end

data/racecar.gemspec

@@ -21,9 +21,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "king_konf", "~> 0.3.7"
-  spec.add_runtime_dependency "rdkafka",   "~> 0.6.0"
+  spec.add_runtime_dependency "rdkafka",   "~> 0.8.0"
 
   spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
+  spec.add_development_dependency "pry"
   spec.add_development_dependency "rake", "> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "timecop"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta4
+  version: 2.1.1
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-02-26 00:00:00.000000000 Z
+date: 2020-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -59,6 +59,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -151,6 +165,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".github/workflows/rspec.yml"
 - ".gitignore"
 - ".rspec"
@@ -162,6 +177,7 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- docker-compose.yml
 - examples/batch_consumer.rb
 - examples/cat_consumer.rb
 - examples/producing_consumer.rb
@@ -203,12 +219,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A framework for running Kafka consumers