karafka 1.2.3 → 1.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dbcc16b7a1dadd11283c52b9e9edd838a0f793e23aebf990aba78baf24c6efa1
-  data.tar.gz: a4da64f172baf0700faae318648b3007ef8406ce861122aa13104daae06213e6
+  metadata.gz: 76f863235897ae7dc92ee3d8353acb6cc3593cffe609dab00b70af78b4895d49
+  data.tar.gz: 8aaa8ee35a193b16fd34029e44e9a4063255e765f236d8268b22d644fcde4675
 SHA512:
-  metadata.gz: b23b92b6c084d934cf64425f0d6a88ec63300972dd1f6c047f8e99be5e7f6233d2ca89af37d562fd9acee2025f60f8fdf4159bff1222ccb624c5b61526696922
-  data.tar.gz: d0ff78e7e938070b95d37b38cde37729a28cf26cc1bb47706fe848a8fe18a5a58f078b1f384e3ecefdcc07861cc4a18e66a68e8c674116f5e773acf2ea8988bb
+  metadata.gz: 26f5aa227da31b07612cd363bcdcccc4fec974061874fdd9878fd6afa9cd952a8901d83d19dfea9130f7c34e3704bb57b3e5d974b7875ad6c0027f50c7ab78ce
+  data.tar.gz: 9bae8c9fc22f27d4e1713d269a37b490f3d461ca2b3fe6fdfda3ae2adebc69d284441860d33c0ceb7a67be7e215e05896da434cbaaebd069c25e4ca90865c7df

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Karafka framework changelog
 
+## 1.2.4
+- #332 - Fetcher for max queue size
+
 ## 1.2.3
 - #313 - support PLAINTEXT and SSL for scheme
 - #320 - Pausing indefinetely with nil pause timeout doesn't work

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (1.2.3)
+    karafka (1.2.4)
       activesupport (>= 4.0)
       dry-configurable (~> 0.7)
       dry-inflector (~> 0.1.1)
@@ -73,7 +73,7 @@ GEM
       concurrent-ruby (~> 1.0)
     inflecto (0.0.2)
     json (2.1.0)
-    king_konf (0.2.0)
+    king_konf (0.3.2)
     minitest (5.11.3)
     multi_json (1.13.1)
     null-logger (0.1.5)
@@ -93,7 +93,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.1)
-    ruby-kafka (0.6.2)
+    ruby-kafka (0.6.3)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)

data/lib/karafka/attributes_map.rb

@@ -18,7 +18,7 @@ module Karafka
         {
           consumer: %i[
             session_timeout offset_commit_interval offset_commit_threshold
-            offset_retention_time heartbeat_interval
+            offset_retention_time heartbeat_interval fetcher_max_queue_size
           ],
           subscribe: %i[start_from_beginning max_bytes_per_partition],
           consumption: %i[min_bytes max_bytes max_wait_time],

data/lib/karafka/schemas/consumer_group.rb

@@ -36,6 +36,7 @@ module Karafka
       required(:offset_commit_threshold).filled(:int?)
       required(:offset_retention_time) { none?.not > int? }
       required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
+      required(:fetcher_max_queue_size).filled(:int?, gt?: 0)
       required(:connect_timeout).filled { (int? | float?) & gt?(0) }
       required(:socket_timeout).filled { (int? | float?) & gt?(0) }
       required(:min_bytes).filled(:int?, gt?: 0)

data/lib/karafka/setup/config.rb

@@ -69,14 +69,14 @@ module Karafka
         setting :seed_brokers
         # option session_timeout [Integer] the number of seconds after which, if a client
         #   hasn't contacted the Kafka cluster, it will be kicked out of the group.
+        setting :pause_timeout, 10
+        # option offset_commit_interval [Integer] the interval between offset commits,
+        #   in seconds.
         setting :session_timeout, 30
         # Time that a given partition will be paused from fetching messages, when message
         # consumption fails. It allows us to process other partitions, while the error is being
         # resolved and also "slows" things down, so it prevents from "eating" up all messages and
         # consuming them with failed code. Use `nil` if you want to pause forever and never retry.
-        setting :pause_timeout, 10
-        # option offset_commit_interval [Integer] the interval between offset commits,
-        #   in seconds.
         setting :offset_commit_interval, 10
         # option offset_commit_threshold [Integer] the number of messages that can be
         #   processed before their offsets are committed. If zero, offset commits are
@@ -85,6 +85,13 @@ module Karafka
         # option heartbeat_interval [Integer] the interval between heartbeats; must be less
         #   than the session window.
         setting :heartbeat_interval, 10
+        # option offset_retention_time [Integer] The length of the retention window, known as
+        #   offset retention time
+        setting :offset_retention_time, nil
+        # option fetcher_max_queue_size [Integer] max number of items in the fetch queue that
+        #   are stored for further processing. Note, that each item in the queue represents a
+        #   response from a single broker
+        setting :fetcher_max_queue_size, 100
         # option max_bytes_per_partition [Integer] the maximum amount of data fetched
         #   from a single partition at a time.
         setting :max_bytes_per_partition, 1_048_576
@@ -111,9 +118,6 @@ module Karafka
         # option reconnect_timeout [Integer] How long should we wait before trying to reconnect to
         # Kafka cluster that went down (in seconds)
         setting :reconnect_timeout, 5
-        # option offset_retention_time [Integer] The length of the retention window, known as
-        #   offset retention time
-        setting :offset_retention_time, nil
         # option connect_timeout [Integer] Sets the number of seconds to wait while connecting to
         # a broker for the first time. When ruby-kafka initializes, it needs to connect to at
         # least one host.

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.2.3'
+  VERSION = '1.2.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.2.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-09 00:00:00.000000000 Z
+date: 2018-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport