racecar 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dbf41d42dfa928893b72e569837c86cdab99e180a4c182c6b7ef7194553f57f5
-  data.tar.gz: 4e67b732e59c59431c34a19d32b0a5d0ffae64a31fd858e35d2673b26c3ac192
+  metadata.gz: 90dddff7c52ef8440ab154be364aacd71f687e7121eaa3456e6b580c3717c294
+  data.tar.gz: '092eb0699d0e60537840cd0114497003ee32e18d611a5b386d430fdfeabd7e03'
 SHA512:
-  metadata.gz: a501912a2c4f9f8d32c943b1c9516042998b241ce743449c3db48623507e360bc14af96c70a2f3b1448550de9e5e6a45d917bf31b3a2f6270f4401eed9c1d70a
-  data.tar.gz: 8e13043d7f41ff8fcaa47d7e5b1cdd4f5a2fd75eba04579172e31dbe76c7831a0d8fc9ed1dd976baec2a8d782c6ad4656f96709d7b3161f7186004aa633a5324
+  metadata.gz: cd4d053961a7f228aeda34bb01e758f9caa6356cf4fb7a3e773c14b9976c19ab30587ce23a010e31b115f30d85999bb38c120e0313b99b43cc744b71f273fb3c
+  data.tar.gz: d0d1865ef2c7adcf94e1bf9c551d7bcdc02a223ee50fbfe4e7293b89629ca7853c55c5cfe144ce879c8aa28c3b8939dcd87fc01c54d441369cfe3c4a186643f3

data/.circleci/config.yml

@@ -0,0 +1,56 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@0.1.2
+
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.6.3-stretch-node
+    executor: ruby/default
+    steps:
+      - checkout
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - ruby/bundle-install
+      - run: bundle exec rspec --exclude-pattern='spec/integration/*_spec.rb'
+  integration-tests:
+    docker:
+      - image: circleci/ruby:2.6.3-stretch-node
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_PORT: 9092
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9093
+          KAFKA_PORT: 9093
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9094
+          KAFKA_PORT: 9094
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+    executor: ruby/default
+    steps:
+      - checkout
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - ruby/bundle-install
+      - run: bundle exec rspec --pattern='spec/integration/*_spec.rb'
+
+workflows:
+  version: 2
+  test:
+    jobs:
+      - build
+      - integration-tests

data/CHANGELOG.md

@@ -1,8 +1,14 @@
 # Changelog
 
+## Unreleased
+
+## racecar v2.1.0
+
+* Bump rdkafka to 0.8.0 (#191)
+
 ## racecar v2.0.0
 
-* Replace `ruby-kafka` with `rdkafka-ruby` as the low-level library underneath Racecar.
+* Replace `ruby-kafka` with `rdkafka-ruby` as the low-level library underneath Racecar (#91).
 * Fix `max_wait_time` usage (#179).
 * Removed config option `sasl_over_ssl`.
 * [Racecar::Consumer] Do not pause consuming partitions on exception.
@@ -26,6 +32,8 @@
 * [Instrumentation] if processors define a `statistics_callback`, it will be called once every second for every subscription or producer connection. The first argument will be a Hash, for contents see [librdkafka STATISTICS.md](https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md).
 * Add current directory to `$LOAD_PATH` only when `--require` option is used (#117).
 * Remove manual heartbeat support, see [Long-running message processing section in README](README.md#long-running-message-processing).
+* Rescue exceptions--then log and pass to `on_error`--at the outermost level of `exe/racecar`, so that exceptions raised outside `Cli.run` are not silently discarded (#186).
+* When exceptions with a `cause` are logged, recursively log the `cause` detail, separated by `--- Caused by: ---\n`.
 
 ## racecar v1.0.0
 

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in racecar.gemspec

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 

data/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '2'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.5.1
+    hostname: zookeeper
+    container_name: zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  broker:
+    image: confluentinc/cp-kafka:5.5.1
+    hostname: broker
+    container_name: broker
+    depends_on:
+      - zookeeper
+    ports:
+      - "29092:29092"
+      - "9092:9092"
+      - "9101:9101"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_JMX_PORT: 9101

data/examples/batch_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class BatchConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/examples/cat_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class CatConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/examples/producing_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ProducingConsumer < Racecar::Consumer
   subscribes_to "messages", start_from_beginning: false
 

data/exe/racecar

@@ -3,19 +3,42 @@
 require "racecar"
 require "racecar/cli"
 
-begin
-  Racecar::Cli.main(ARGV)
-rescue SignalException => e
-  # We might receive SIGTERM before our signal handler is installed.
-  if Signal.signame(e.signo) == "TERM"
-    exit(0)
-  else
-    raise
+module Racecar
+  class << self
+    def start(argv)
+      Cli.main(argv)
+    rescue SignalException => e
+      # We might receive SIGTERM before our signal handler is installed.
+      if Signal.signame(e.signo) == "TERM"
+        exit(0)
+      else
+        raise
+      end
+    rescue SystemExit
+      raise
+    rescue Exception => e
+      $stderr.puts "=> Crashed: #{exception_with_causes(e)}\n#{e.backtrace.join("\n")}"
+
+      Racecar.config.error_handler.call(e)
+
+      exit(1)
+    else
+      exit(0)
+    end
+
+    private
+
+    def exception_with_causes(e)
+      result = +"#{e.class}: #{e}"
+      if e.cause
+        result << "\n"
+        result << "--- Caused by: ---\n"
+        result << exception_with_causes(e.cause)
+      end
+      result
+    end
   end
-rescue
-  # Exceptions are printed to STDERR and sent to the error handler
-  # in `Racecar::Cli#run`, so we don't need to do anything here.
-  exit(1)
-else
-  exit(0)
 end
+
+# Start your engines!
+Racecar.start(ARGV)

data/lib/ensure_hash_compact.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # only needed when ruby < 2.4 and not using active support
 
 unless {}.respond_to? :compact

data/lib/generators/racecar/consumer_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module Generators
     class ConsumerGenerator < Rails::Generators::NamedBase

data/lib/generators/racecar/install_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module Generators
     class InstallGenerator < Rails::Generators::Base

data/lib/racecar.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "logger"
 
 require "racecar/instrumenter"
@@ -6,6 +8,7 @@ require "racecar/consumer"
 require "racecar/consumer_set"
 require "racecar/runner"
 require "racecar/config"
+require "racecar/version"
 require "ensure_hash_compact"
 
 module Racecar

data/lib/racecar/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "optparse"
 require "logger"
 require "fileutils"
@@ -6,8 +8,10 @@ require "racecar/daemon"
 
 module Racecar
   class Cli
-    def self.main(args)
-      new(args).run
+    class << self
+      def main(args)
+        new(args).run
+      end
     end
 
     def initialize(args)
@@ -16,10 +20,6 @@ module Racecar
       @consumer_name = args.first or raise Racecar::Error, "no consumer specified"
     end
 
-    def config
-      Racecar.config
-    end
-
     def run
       $stderr.puts "=> Starting Racecar consumer #{consumer_name}..."
 
@@ -61,18 +61,16 @@ module Racecar
       processor = consumer_class.new
 
       Racecar.run(processor)
-    rescue => e
-      $stderr.puts "=> Crashed: #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
-
-      config.error_handler.call(e)
-
-      raise
     end
 
     private
 
     attr_reader :consumer_name
 
+    def config
+      Racecar.config
+    end
+
     def daemonize!
       daemon = Daemon.new(File.expand_path(config.pidfile))
 
@@ -102,12 +100,7 @@ module Racecar
         opts.on("-r", "--require STRING", "Require a library before starting the consumer") do |lib|
           $LOAD_PATH.unshift(Dir.pwd) unless load_path_modified
           load_path_modified = true
-          begin
-            require lib
-          rescue => e
-            $stderr.puts "=> #{lib} failed to load: #{e.message}"
-            exit
-          end
+          require lib
         end
 
         opts.on("-l", "--log STRING", "Log to the specified file") do |logfile|
@@ -115,13 +108,13 @@ module Racecar
         end
 
         Racecar::Config.variables.each do |variable|
-          opt_name = "--" << variable.name.to_s.gsub("_", "-")
+          opt_name = +"--#{variable.name.to_s.gsub('_', '-')}"
           opt_name << " #{variable.type.upcase}" unless variable.boolean?
 
           desc = variable.description || "N/A"
 
           if variable.default
-            desc << " (default: #{variable.default.inspect})"
+            desc += " (default: #{variable.default.inspect})"
           end
 
           opts.on(opt_name, desc) do |value|

data/lib/racecar/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "king_konf"
 
 module Racecar
@@ -196,8 +198,8 @@ module Racecar
         group_id_prefix,
 
         # MyFunnyConsumer => my-funny-consumer
-        consumer_class.name.gsub(/[a-z][A-Z]/) {|str| str[0] << "-" << str[1] }.downcase,
-      ].compact.join("")
+        consumer_class.name.gsub(/[a-z][A-Z]/) { |str| "#{str[0]}-#{str[1]}" }.downcase,
+      ].compact.join
 
       self.subscriptions = consumer_class.subscriptions
       self.max_wait_time = consumer_class.max_wait_time || self.max_wait_time

data/lib/racecar/consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   class Consumer
     Subscription = Struct.new(:topic, :start_from_beginning, :max_bytes_per_partition, :additional_config)
@@ -54,7 +56,7 @@ module Racecar
     protected
 
     # https://github.com/appsignal/rdkafka-ruby#producing-messages
-    def produce(payload, topic:, key:, partition_key: nil, headers: nil, create_time: nil)
+    def produce(payload, topic:, key: nil, partition_key: nil, headers: nil, create_time: nil)
       @delivery_handles ||= []
       message_size = payload.respond_to?(:bytesize) ? payload.bytesize : 0
       instrumentation_payload = {

data/lib/racecar/consumer_set.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   class ConsumerSet
     MAX_POLL_TRIES = 10

data/lib/racecar/ctl.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "optparse"
 require "racecar/rails_config_file_loader"
 require "racecar/daemon"

data/lib/racecar/daemon.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   class Daemon
     attr_reader :pidfile

data/lib/racecar/datadog.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 begin
   require "datadog/statsd"
 rescue LoadError

data/lib/racecar/instrumenter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   ##
   # Common API for instrumentation to standardize
@@ -11,6 +13,8 @@ module Racecar
       @default_payload = default_payload
 
       @backend = if defined?(ActiveSupport::Notifications)
+        # ActiveSupport needs `concurrent-ruby` but doesn't `require` it.
+        require 'concurrent/utility/monotonic_time'
         ActiveSupport::Notifications
       else
         NullInstrumenter

data/lib/racecar/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "forwardable"
 
 module Racecar

data/lib/racecar/null_instrumenter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   # Ignores all instrumentation events.
   class NullInstrumenter

data/lib/racecar/pause.rb

@@ -1,8 +1,12 @@
+# frozen_string_literal: true
+
 module Racecar
   class Pause
+    attr_reader :pauses_count
+
     def initialize(timeout: nil, max_timeout: nil, exponential_backoff: false)
       @started_at = nil
-      @pauses = 0
+      @pauses_count = 0
       @timeout = timeout
       @max_timeout = max_timeout
       @exponential_backoff = exponential_backoff
@@ -11,7 +15,7 @@ module Racecar
     def pause!
       @started_at = Time.now
       @ends_at = @started_at + backoff_interval unless @timeout.nil?
-      @pauses += 1
+      @pauses_count += 1
     end
 
     def resume!
@@ -38,13 +42,13 @@ module Racecar
     end
 
     def reset!
-      @pauses = 0
+      @pauses_count = 0
     end
 
     def backoff_interval
       return Float::INFINITY if @timeout.nil?
 
-      backoff_factor = @exponential_backoff ? 2**@pauses : 1
+      backoff_factor = @exponential_backoff ? 2**@pauses_count : 1
       timeout = backoff_factor * @timeout
 
       timeout = @max_timeout if @max_timeout && timeout > @max_timeout

data/lib/racecar/rails_config_file_loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
   module RailsConfigFileLoader
     def self.load!

data/lib/racecar/runner.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "rdkafka"
 require "racecar/pause"
 require "racecar/message"
@@ -166,7 +168,7 @@ module Racecar
       }
 
       @instrumenter.instrument("start_process_message", instrumentation_payload)
-      with_pause(message.topic, message.partition, message.offset..message.offset) do
+      with_pause(message.topic, message.partition, message.offset..message.offset) do |pause|
         begin
           @instrumenter.instrument("process_message", instrumentation_payload) do
             processor.process(Racecar::Message.new(message))
@@ -174,6 +176,7 @@ module Racecar
             consumer.store_offset(message)
           end
         rescue => e
+          instrumentation_payload[:retries_count] = pause.pauses_count
           config.error_handler.call(e, instrumentation_payload)
           raise e
         end
@@ -194,12 +197,13 @@ module Racecar
 
       @instrumenter.instrument("start_process_batch", instrumentation_payload)
       @instrumenter.instrument("process_batch", instrumentation_payload) do
-        with_pause(first.topic, first.partition, first.offset..last.offset) do
+        with_pause(first.topic, first.partition, first.offset..last.offset) do |pause|
           begin
             processor.process_batch(messages.map {|message| Racecar::Message.new(message) })
             processor.deliver!
             consumer.store_offset(messages.last)
           rescue => e
+            instrumentation_payload[:retries_count] = pause.pauses_count
             config.error_handler.call(e, instrumentation_payload)
             raise e
           end
@@ -208,17 +212,17 @@ module Racecar
     end
 
     def with_pause(topic, partition, offsets)
-      return yield if config.pause_timeout == 0
+      pause = pauses[topic][partition]
+      return yield pause if config.pause_timeout == 0
 
       begin
-        yield
+        yield pause
         # We've successfully processed a batch from the partition, so we can clear the pause.
         pauses[topic][partition].reset!
       rescue => e
         desc = "#{topic}/#{partition}"
         logger.error "Failed to process #{desc} at #{offsets}: #{e}"
 
-        pause = pauses[topic][partition]
         logger.warn "Pausing partition #{desc} for #{pause.backoff_interval} seconds"
         consumer.pause(topic, partition, offsets.first)
         pause.pause!

data/lib/racecar/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Racecar
-  VERSION = "2.0.0"
+  VERSION = "2.1.0"
 end

data/racecar.gemspec

@@ -21,9 +21,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "king_konf", "~> 0.3.7"
-  spec.add_runtime_dependency "rdkafka",   "~> 0.8.0.beta.1"
+  spec.add_runtime_dependency "rdkafka",   "~> 0.8.0"
 
   spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
+  spec.add_development_dependency "pry"
   spec.add_development_dependency "rake", "> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "timecop"

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 - Benjamin Quorning
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-24 00:00:00.000000000 Z
+date: 2020-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0.beta.1
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0.beta.1
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -59,6 +59,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -141,7 +155,7 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '6.1'
-description:
+description: 
 email:
 - dschierbeck@zendesk.com
 - bquorning@zendesk.com
@@ -151,6 +165,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".github/workflows/rspec.yml"
 - ".gitignore"
 - ".rspec"
@@ -162,6 +177,7 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- docker-compose.yml
 - examples/batch_consumer.rb
 - examples/cat_consumer.rb
 - examples/producing_consumer.rb
@@ -192,7 +208,7 @@ homepage: https://github.com/zendesk/racecar
 licenses:
 - Apache License Version 2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -208,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key:
+signing_key: 
 specification_version: 4
 summary: A framework for running Kafka consumers
 test_files: []