racecar 2.1.1 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6380c598cedfca4662aa4c0edf6c55b8a0641d972dad715e3b1e961ac06e4d4f
-  data.tar.gz: 19a30f8515b82f9cbfa783319a76059688ef6096a4dd5e5a5673d6d0c101878c
+  metadata.gz: e2c32338556d27bfcbae35df0758163a457d2c6e3f520141206c5a9c0124cc68
+  data.tar.gz: 43d9c42d0d483c269b15d417ef59e2985da3ca3945d34d6772f8dd262ccbfaf6
 SHA512:
-  metadata.gz: 66c6013d77d63a121673e7d4e49fb87b3b1520a3e80a9b564a116b284dbd262ead6fd5b570afa4fa55d77822ac6a6ab31a86917b08630aad2fc9af2f7c08d171
-  data.tar.gz: ce9c89a671478a806dce62979bb695a2673cd598acfb3d2547194a9367a12c96d5f34d38c32f79a3ae809f9eb79303e5330aec4f5e0dc2a23f6f6ffc1e286817
+  metadata.gz: 0467ac1cdefb6cad9870dd73b92f4a5a943b9f685ff3fc876b3f183d109ae3d29d7c2c7dffea8f31bca7c7b18565e5aba04d4865c94f2448a7228be175855a5b
+  data.tar.gz: e4ab43eb180995af916d447b006438b4a48cb808b29aabec52b455e246541a083192d1b560a957fa6f3ab7d5412dd12ab74aac49acc0b606c3df87cec90b93b6

data/.github/workflows/ci.yml

@@ -0,0 +1,61 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - '**'
+
+jobs:
+  unit-specs:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        ruby-version: ["2.5", "2.6"]
+
+    steps:
+    - uses: zendesk/checkout@v2
+    - name: Set up Ruby
+      uses: zendesk/setup-ruby@v1.58.0
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true
+    - name: Build and test with RSpec
+      run: bundle exec rspec --format documentation --require spec_helper --color --exclude-pattern='spec/integration/*_spec.rb'
+
+  integration-specs:
+    runs-on: ubuntu-latest
+
+    services:
+      zookeeper:
+        image: confluentinc/cp-zookeeper
+        ports:
+          - 2181:2181
+        env:
+          ZOOKEEPER_CLIENT_PORT: 2181
+
+      kafka:
+        image: confluentinc/cp-kafka
+        ports:
+          - 9092:9092
+          - 29092:29092
+        options: --health-cmd "kafka-topics --list --bootstrap-server=localhost:9092" --health-interval 10s --health-timeout 5s --health-retries 5
+        env:
+          KAFKA_BROKER_ID: 1
+          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+          KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+
+    steps:
+    - uses: zendesk/checkout@v2
+    - name: Set up Ruby
+      uses: zendesk/setup-ruby@v1.58.0
+      with:
+        ruby-version: 2.7
+        bundler-cache: true
+    - name: Build and test with RSpec
+      env:
+        RACECAR_BROKERS: localhost:9092
+      run: bundle exec rspec --format documentation --require spec_helper --color spec/integration/*_spec.rb

data/.gitignore

@@ -1,10 +1,9 @@
 /.bundle/
 /.yardoc
-/Gemfile.lock
 /_yardoc/
 /coverage/
 /doc/
 /pkg/
 /spec/reports/
 /tmp/
-/vendor/bundle/
+/vendor/bundle/

data/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+## racecar v2.2.0
+
+* [Racecar::ConsumerSet] **breaking change** `Racecar::ConsumerSet`'s functions `poll` and `batch_pall` expect the max wait values to be given in milliseconds. The defaults were using `config.max_wait_time`, which is in seconds. If you do not directly use `Racecar::ConsumerSet`, or always call its `poll` and `batch_poll` functions by specfiying the max wait time (the first argument), then this breaking change does not affect you. ([#214](https://github.com/zendesk/racecar/pull/214))
+
 ## racecar v2.1.1
 
 * [Bugfix] Close RdKafka consumer in ConsumerSet#reset_current_consumer to prevent memory leak (#196)

data/Gemfile.lock

@@ -0,0 +1,69 @@
+PATH
+  remote: .
+  specs:
+    racecar (2.1.1)
+      king_konf (~> 1.0.0)
+      rdkafka (~> 0.8.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.0.3.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.7)
+    diff-lcs (1.4.4)
+    dogstatsd-ruby (4.8.2)
+    ffi (1.13.1)
+    i18n (1.8.5)
+      concurrent-ruby (~> 1.0)
+    king_konf (1.0.0)
+    method_source (1.0.0)
+    mini_portile2 (2.5.0)
+    minitest (5.14.2)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (13.0.1)
+    rdkafka (0.8.1)
+      ffi (~> 1.9)
+      mini_portile2 (~> 2.1)
+      rake (>= 12.3)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.0)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.0)
+    thread_safe (0.3.6)
+    timecop (0.9.2)
+    tzinfo (1.2.8)
+      thread_safe (~> 0.1)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (>= 4.0, < 6.1)
+  bundler (>= 1.13, < 3)
+  dogstatsd-ruby (>= 4.0.0, < 5.0.0)
+  pry
+  racecar!
+  rake (> 10.0)
+  rspec (~> 3.0)
+  timecop
+
+BUNDLED WITH
+   2.1.4

data/lib/racecar/config.rb

@@ -54,7 +54,7 @@ module Racecar
     desc "How long to wait when trying to communicate with a Kafka broker"
     float :socket_timeout, default: 30
 
-    desc "How long to allow the Kafka brokers to wait before returning messages"
+    desc "How long to allow the Kafka brokers to wait before returning messages (in seconds)"
     float :max_wait_time, default: 1
 
     desc "Maximum amount of data the broker shall return for a Fetch request"

data/lib/racecar/consumer_set.rb

@@ -17,7 +17,7 @@ module Racecar
       @last_poll_read_nil_message = false
     end
 
-    def poll(max_wait_time_ms = @config.max_wait_time)
+    def poll(max_wait_time_ms = @config.max_wait_time_ms)
       batch_poll(max_wait_time_ms, 1).first
     end
 
@@ -31,7 +31,7 @@ module Racecar
     # Any errors during polling are retried in an exponential backoff fashion. If an error
     # occurs, but there is no time left for a backoff and retry, it will return the
     # already collected messages and only retry on the next call.
-    def batch_poll(max_wait_time_ms = @config.max_wait_time, max_messages = @config.fetch_messages)
+    def batch_poll(max_wait_time_ms = @config.max_wait_time_ms, max_messages = @config.fetch_messages)
       started_at = Time.now
       remain_ms = max_wait_time_ms
       maybe_select_next_consumer
@@ -130,6 +130,9 @@ module Racecar
         @logger.debug "Capping #{wait_ms}ms to #{max_wait_time_ms-1}ms."
         sleep (max_wait_time_ms-1)/1000.0
         remain_ms = 1
+      elsif try == 0 && remain_ms == 0
+        @logger.debug "No time remains for polling messages. Will try on next call."
+        return nil
       elsif wait_ms >= remain_ms
         @logger.error "Only #{remain_ms}ms left, but want to wait for #{wait_ms}ms before poll. Will retry on next call."
         @previous_retries = try

data/lib/racecar/message.rb

@@ -6,8 +6,11 @@ module Racecar
   class Message
     extend Forwardable
 
-    def initialize(rdkafka_message)
+    attr_reader :retries_count
+
+    def initialize(rdkafka_message, retries_count: nil)
       @rdkafka_message = rdkafka_message
+      @retries_count   = retries_count
     end
 
     def_delegators :@rdkafka_message, :topic, :partition, :offset, :key, :headers

data/lib/racecar/runner.rb

@@ -171,7 +171,7 @@ module Racecar
       with_pause(message.topic, message.partition, message.offset..message.offset) do |pause|
         begin
           @instrumenter.instrument("process_message", instrumentation_payload) do
-            processor.process(Racecar::Message.new(message))
+            processor.process(Racecar::Message.new(message, retries_count: pause.pauses_count))
             processor.deliver!
             consumer.store_offset(message)
           end
@@ -199,7 +199,10 @@ module Racecar
       @instrumenter.instrument("process_batch", instrumentation_payload) do
         with_pause(first.topic, first.partition, first.offset..last.offset) do |pause|
           begin
-            processor.process_batch(messages.map {|message| Racecar::Message.new(message) })
+            racecar_messages = messages.map do |message|
+              Racecar::Message.new(message, retries_count: pause.pauses_count)
+            end
+            processor.process_batch(racecar_messages)
             processor.deliver!
             consumer.store_offset(messages.last)
           rescue => e

data/lib/racecar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Racecar
-  VERSION = "2.1.1"
+  VERSION = "2.2.0"
 end

data/racecar.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "king_konf", "~> 0.3.7"
+  spec.add_runtime_dependency "king_konf", "~> 1.0.0"
   spec.add_runtime_dependency "rdkafka",   "~> 0.8.0"
 
   spec.add_development_dependency "bundler", [">= 1.13", "< 3"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-04 00:00:00.000000000 Z
+date: 2021-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.7
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.7
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: rdkafka
   requirement: !ruby/object:Gem::Requirement
@@ -166,11 +166,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
-- ".github/workflows/rspec.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - Procfile
 - README.md

data/.github/workflows/rspec.yml

@@ -1,24 +0,0 @@
-name: Execute Specs
-
-on: [push]
-
-jobs:
-  rspec:
-
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        ruby-version: ["2.5.x", "2.6.x"]
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Ruby 2.6
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby-version }}
-    - name: Build and test with RSpec
-      run: |
-        gem install bundler --no-document
-        bundle install --jobs 4 --retry 3
-        bundle exec rspec --format documentation --require spec_helper --color