racecar 2.0.0.alpha3 → 2.0.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb4a01b20ccfb91b90a44a87a51acfc9c60a484450a86b47e73a955761a41365
-  data.tar.gz: 73dfe494f6dd8b867f45b8346f0d2dce8ba91e56e40e83dbf967fb4a8f2c1841
+  metadata.gz: 1def66ffdef1ce2d2c100dea3b74a3d69fc5c17ef0c57071ef25cd7cbe4016ad
+  data.tar.gz: 8198710b789bfa1d49291b020e652e2ab5a7a80e17ccb4ef6a653b3416702342
 SHA512:
-  metadata.gz: 5b2d334a5e2564c79ba4e1f91ba6ffd1e44001f52ec738d47e4c98096b0d76f6f203d14712b393739b1edbfc2af0e66f7809ed7018e63a777765e2e0f577eca5
-  data.tar.gz: c3ccceb024017953c57645678c1fa32a943d8a7dfdf2808d5066e22731ba75547483ddca686e2d00e9a1d5e6816cdd6b3d434d3723ce7722a8a3b7cf57410b5f
+  metadata.gz: f4329d9e10df97ce40c90badf31b65db2379a568b96f90e311e407ce866bbb953e5fccb0a55bf16c006424fe7ecf1e6a6901e513060795f9fabfa7e5b7d36a55
+  data.tar.gz: b04998e5e3786d1c7ab6d9ba9364643709e47f9ba9f753df59edfa3e967e312755b429c47198e10950ee899e972d8330d9aa414849199501de1d41657da45e97

data/.github/workflows/{ci.yml → rspec.yml}

@@ -3,7 +3,7 @@ name: Execute Specs
 on: [push]
 
 jobs:
-  build:
+  rspec:
 
     runs-on: ubuntu-latest
 

data/README.md

@@ -214,7 +214,7 @@ class GeoCodingConsumer < Racecar::Consumer
 
     # The `produce` method enqueues a message to be delivered after #process
     # returns. It won't actually deliver the message.
-    produce(payload: JSON.dump(pageview), topic: "pageviews-with-country", key: pageview["id"])
+    produce(JSON.dump(pageview), topic: "pageviews-with-country", key: pageview["id"])
   end
 end
 ```
@@ -319,6 +319,9 @@ Racecar supports configuring ruby-kafka's [Datadog](https://www.datadoghq.com/)
 * `datadog_namespace` – The namespace to use for Datadog metrics.
 * `datadog_tags` – Tags that should always be set on Datadog metrics.
 
+#### Consumers Without Rails ####
+
+By default, if Rails is detected, it will be automatically started when the consumer is started. There are cases where you might not want or need Rails. You can pass the `--without-rails` option when starting the consumer and Rails won't be started.
 
 ### Testing consumers
 

data/lib/racecar.rb

@@ -18,6 +18,10 @@ module Racecar
     @config ||= Config.new
   end
 
+  def self.config=(config)
+    @config = config
+  end
+
   def self.configure
     yield config
   end

data/lib/racecar/cli.rb

@@ -23,7 +23,7 @@ module Racecar
     def run
       $stderr.puts "=> Starting Racecar consumer #{consumer_name}..."
 
-      RailsConfigFileLoader.load!
+      RailsConfigFileLoader.load! unless config.without_rails?
 
       if File.exist?("config/racecar.rb")
         require "./config/racecar"

data/lib/racecar/config.rb

@@ -136,6 +136,9 @@ module Racecar
     desc "Whether to check the server certificate is valid for the hostname"
     boolean :ssl_verify_hostname, default: true
 
+    desc "Whether to boot Rails when starting the consumer"
+    boolean :without_rails, default: false
+
     # The error handler must be set directly on the object.
     attr_reader :error_handler
 

data/lib/racecar/consumer_set.rb

@@ -1,5 +1,7 @@
 module Racecar
   class ConsumerSet
+    MAX_POLL_TRIES = 10
+
     def initialize(config, logger)
       @config, @logger = config, logger
       raise ArgumentError, "Subscriptions must not be empty when subscribing" if @config.subscriptions.empty?
@@ -12,21 +14,28 @@ module Racecar
 
     def poll(timeout_ms)
       maybe_select_next_consumer
-      retried ||= false
-      msg = current.poll(timeout_ms)
+      started_at ||= Time.now
+      try ||= 0
+      remain ||= timeout_ms
+
+      msg = remain <= 0 ? nil : current.poll(remain)
     rescue Rdkafka::RdkafkaError => e
-      raise if retried
-      retried = true
+      wait_before_retry_ms = 100 * (2**try) # 100ms, 200ms, 400ms, …
+      try += 1
+      raise if try >= MAX_POLL_TRIES || remain <= wait_before_retry_ms
 
-      @logger.error "Error for topic subscription #{current_subscription}: #{e}"
+      @logger.error "(try #{try}): Error for topic subscription #{current_subscription}: #{e}"
 
       case e.code
       when :max_poll_exceeded, :transport # -147, -195
         reset_current_consumer
-        retry
-      else
-        raise
       end
+
+      remain = remaining_time_ms(timeout_ms, started_at)
+      raise if remain <= wait_before_retry_ms
+
+      sleep wait_before_retry_ms/1000.0
+      retry
     ensure
       @last_poll_read_nil_message = true if msg.nil?
     end
@@ -36,7 +45,9 @@ module Racecar
       @batch_started_at = Time.now
       @messages = []
       while collect_messages_for_batch? do
-        msg = poll(timeout_ms)
+        remain = remaining_time_ms(timeout_ms, @batch_started_at)
+        break if remain <= 0
+        msg = poll(remain)
         break if msg.nil?
         @messages << msg
       end
@@ -174,5 +185,10 @@ module Racecar
       config.merge! subscription.additional_config
       config
     end
+
+    def remaining_time_ms(limit_ms, started_at_time)
+      r = limit_ms - ((Time.now - started_at_time)*1000).round
+      r <= 0 ? 0 : r
+    end
   end
 end

data/lib/racecar/message.rb

@@ -8,7 +8,11 @@ module Racecar
       @rdkafka_message = rdkafka_message
     end
 
-    def_delegators :@rdkafka_message, :topic, :partition, :offset, :key, :value, :headers
+    def_delegators :@rdkafka_message, :topic, :partition, :offset, :key, :headers
+
+    def value
+      @rdkafka_message.payload
+    end
 
     def create_time
       @rdkafka_message.timestamp

data/lib/racecar/version.rb

@@ -1,3 +1,3 @@
 module Racecar
-  VERSION = "2.0.0.alpha3"
+  VERSION = "2.0.0.beta1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.0.0.alpha3
+  version: 2.0.0.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-09-18 00:00:00.000000000 Z
+date: 2020-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -111,7 +111,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/ci.yml"
+- ".github/workflows/rspec.yml"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md