karafka 1.4.2 → 1.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b67c595b8ce4b5ff1589517547f94c8e1ba4caf51361074949b7bae1c5368bf6
-  data.tar.gz: 17c507ddf4d2b4cd62eea3df3a1dec0cef5a257054f7d0321e1c71dcba0d400e
+  metadata.gz: fc0edb1580179b926b7adaed4901c8483e0cfa1094b8951adecda7105a2fa1ec
+  data.tar.gz: 4392bc711588e38fe53c6b6a9966e7226652efd6db8bfc8b48b7f845bd950a28
 SHA512:
-  metadata.gz: 818d4d75303eda22f8e923e7ade1c445f0c2e99aebdb7e56511b5fdf38ea39c047f61c0983bc8d38641676be2c2fcc901df9d2bc0d812c3454664dd73ab27b2e
-  data.tar.gz: 5c10aced279e74af09d42dd3367a1593abf11267cb09099d0d92a3b4e540f19fa3c06bd9e0f61f7025c3fc0774cde9f4902fb929ca9e24aa308f9d72e6a61def
+  metadata.gz: 98c768c6cf9fcb1a594488d4a5f40f87f8c676872f2bee13eec0f20fabb060078594420b2028d57f8aa4c64de2517027df83de9242145e40a9df782e9ec329f0
+  data.tar.gz: 28d54e29cc197926f6fe9856334e9aad1b3013102ef7a0aec0ea7425e92d798fb3dcaece97ed3c84d89e9e4e9f170946745951ad9fdfc1730572c240fd99cf31

data/.github/workflows/ci.yml

@@ -1,6 +1,7 @@
 name: ci
 
 on:
+  pull_request:
   push:
   schedule:
     - cron:  '0 1 * * *'
@@ -16,7 +17,6 @@ jobs:
           - '3.0'
           - '2.7'
           - '2.6'
-          - '2.5'
         include:
           - ruby: '3.0'
             coverage: 'true'

data/.ruby-version

@@ -1 +1 @@
-3.0.0
+3.0.2

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Karafka framework changelog
 
+## 1.4.6 (2021-08-05)
+- #700 Fix Ruby 3 compatibility issues in Connection::Client#pause (MmKolodziej)
+
+## 1.4.5 (2021-06-16)
+- Fixup logger checks for non-writeable logfile (ojab)
+- #689 - Update the stdout initialization message for framework initialization
+
+## 1.4.4 (2021-04-19)
+- Remove Ruby 2.5 support and update minimum Ruby requirement to 2.6
+- Remove rake dependency
+
+## 1.4.3 (2021-03-24)
+- Fixes for Ruby 3.0 compatibility
+
 ## 1.4.2 (2021-02-16)
 - Rescue Errno::EROFS in ensure_dir_exists (unasuke)
 

data/Gemfile.lock

@@ -1,14 +1,13 @@
 PATH
   remote: .
   specs:
-    karafka (1.4.2)
+    karafka (1.4.6)
       dry-configurable (~> 0.8)
       dry-inflector (~> 0.1)
       dry-monitor (~> 0.3)
       dry-validation (~> 1.2)
       envlogic (~> 1.1)
       irb (~> 1.0)
-      rake (>= 11.3)
       ruby-kafka (>= 1.0.0)
       thor (>= 0.20)
       waterdrop (~> 1.4.0)
@@ -17,45 +16,43 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.3)
+    activesupport (6.1.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
     byebug (11.1.3)
-    concurrent-ruby (1.1.8)
+    concurrent-ruby (1.1.9)
     delivery_boy (1.1.0)
       king_konf (~> 1.0)
       ruby-kafka (~> 1.0)
     diff-lcs (1.4.4)
-    digest-crc (0.6.3)
+    digest-crc (0.6.4)
       rake (>= 12.0.0, < 14.0.0)
-    docile (1.3.5)
+    docile (1.4.0)
     dry-configurable (0.12.1)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5.0)
-    dry-container (0.7.2)
+    dry-container (0.8.0)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.5.0)
+    dry-core (0.7.1)
       concurrent-ruby (~> 1.0)
     dry-equalizer (0.3.0)
-    dry-events (0.2.0)
+    dry-events (0.3.0)
       concurrent-ruby (~> 1.0)
-      dry-core (~> 0.4)
-      dry-equalizer (~> 0.2)
-    dry-inflector (0.2.0)
+      dry-core (~> 0.5, >= 0.5)
+    dry-inflector (0.2.1)
     dry-initializer (3.0.4)
-    dry-logic (1.1.0)
+    dry-logic (1.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
-    dry-monitor (0.3.2)
+    dry-monitor (0.4.0)
       dry-configurable (~> 0.5)
-      dry-core (~> 0.4)
-      dry-equalizer (~> 0.2)
+      dry-core (~> 0.5, >= 0.5)
       dry-events (~> 0.2)
-    dry-schema (1.6.1)
+    dry-schema (1.7.0)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.8, >= 0.8.3)
       dry-core (~> 0.5, >= 0.5)
@@ -75,19 +72,19 @@ GEM
       dry-equalizer (~> 0.2)
       dry-initializer (~> 3.0)
       dry-schema (~> 1.5, >= 1.5.2)
-    envlogic (1.1.2)
+    envlogic (1.1.3)
       dry-inflector (~> 0.1)
-    factory_bot (6.1.0)
+    factory_bot (6.2.0)
       activesupport (>= 5.0.0)
-    i18n (1.8.9)
+    i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    io-console (0.5.8)
-    irb (1.3.4)
-      reline (>= 0.1.5)
+    io-console (0.5.9)
+    irb (1.3.6)
+      reline (>= 0.2.5)
     king_konf (1.0.0)
-    minitest (5.14.3)
-    rake (13.0.3)
-    reline (0.2.4)
+    minitest (5.14.4)
+    rake (13.0.6)
+    reline (0.2.6)
       io-console (~> 0.5)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -109,11 +106,11 @@ GEM
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.2)
+    simplecov_json_formatter (0.1.3)
     thor (1.1.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    waterdrop (1.4.0)
+    waterdrop (1.4.2)
       delivery_boy (>= 0.2, < 2.x)
       dry-configurable (~> 0.8)
       dry-monitor (~> 0.3)
@@ -123,8 +120,7 @@ GEM
     zeitwerk (2.4.2)
 
 PLATFORMS
-  ruby
-  x86_64-darwin-19
+  x86_64-linux
 
 DEPENDENCIES
   byebug
@@ -134,4 +130,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.2.11
+   2.2.24

data/README.md

@@ -1,10 +1,10 @@
 ![karafka logo](https://raw.githubusercontent.com/karafka/misc/master/logo/karafka_logotype_transparent2.png)
 
-[![Build Status](https://travis-ci.org/karafka/karafka.svg?branch=master)](https://travis-ci.org/karafka/karafka)
+[![Build Status](https://github.com/karafka/karafka/actions/workflows/ci.yml/badge.svg)](https://github.com/karafka/karafka/actions/workflows/ci.yml)
+[![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
+[![Join the chat at https://gitter.im/karafka/karafka](https://badges.gitter.im/karafka/karafka.svg)](https://gitter.im/karafka/karafka)
 
-**Note**: Documentation presented here refers to Karafka `1.4`.
-
-If you are looking for the documentation for Karafka `1.3.x`, it can be found [here](https://github.com/karafka/wiki/tree/1.3).
+**Note**: We're finishing the new Karafka `2.0` but for now, please use `1.4`. All the documentation presented here refers to `1.4`
 
 ## About Karafka
 
@@ -72,7 +72,7 @@ Karafka framework and Karafka team are __not__ related to Kafka streaming servic
 ## References
 
 * [Karafka framework](https://github.com/karafka/karafka)
-* [Karafka Travis CI](https://travis-ci.org/karafka/karafka)
+* [Karafka GitHub Actions](https://github.com/karafka/karafka/actions)
 * [Karafka Coditsu](https://app.coditsu.io/karafka/repositories/karafka)
 
 ## Note on contributions

data/karafka.gemspec

@@ -23,13 +23,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'dry-validation', '~> 1.2'
   spec.add_dependency 'envlogic', '~> 1.1'
   spec.add_dependency 'irb', '~> 1.0'
-  spec.add_dependency 'rake', '>= 11.3'
   spec.add_dependency 'ruby-kafka', '>= 1.0.0'
   spec.add_dependency 'thor', '>= 0.20'
   spec.add_dependency 'waterdrop', '~> 1.4.0'
   spec.add_dependency 'zeitwerk', '~> 2.1'
 
-  spec.required_ruby_version = '>= 2.5.0'
+  spec.required_ruby_version = '>= 2.6.0'
 
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')

data/lib/karafka/cli/install.rb

@@ -34,7 +34,7 @@ module Karafka
           Bundler.read_file(
             Bundler.default_lockfile
           )
-        ).dependencies.key?('rails')
+        ).dependencies.key?('railties')
       end
 
       # Install all required things for Karafka application in current directory

data/lib/karafka/cli/server.rb

@@ -31,14 +31,6 @@ module Karafka
         # part of the topics
         Karafka::Server.consumer_groups = cli.options[:consumer_groups]
 
-        # Remove pidfile on stop, just before the server instance is going to be GCed
-        # We want to delay the moment in which the pidfile is removed as much as we can,
-        # so instead of removing it after the server stops running, we rely on the gc moment
-        # when this object gets removed (it is a bit later), so it is closer to the actual
-        # system process end. We do that, so monitoring and deployment tools that rely on a pid
-        # won't alarm or start new system process up until the current one is finished
-        ObjectSpace.define_finalizer(self, proc { send(:clean) })
-
         Karafka::Server.run
       end
 
@@ -60,6 +52,14 @@ module Karafka
           cli.options[:pid],
           'w'
         ) { |file| file.write(::Process.pid) }
+
+        # Remove pidfile on stop, just before the server instance is going to be GCed
+        # We want to delay the moment in which the pidfile is removed as much as we can,
+        # so instead of removing it after the server stops running, we rely on the gc moment
+        # when this object gets removed (it is a bit later), so it is closer to the actual
+        # system process end. We do that, so monitoring and deployment tools that rely on a pid
+        # won't alarm or start new system process up until the current one is finished
+        ObjectSpace.define_finalizer(self, proc { send(:clean) })
       end
 
       # Removes a pidfile (if exist)

data/lib/karafka/connection/api_adapter.rb

@@ -48,30 +48,28 @@ module Karafka
 
         # Builds all the configuration settings for kafka#consumer method
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Array<Hash>] array with all the consumer arguments including hash with all
+        # @return [Hash] all the consumer keyword arguments including hash with all
         #   the settings required by Kafka#consumer
         def consumer(consumer_group)
           settings = { group_id: consumer_group.id }
           settings = fetch_for(:consumer, consumer_group, settings)
-          [sanitize(settings)]
+          sanitize(settings)
         end
 
         # Builds all the configuration settings for kafka consumer consume_each_batch and
         #   consume_each_message methods
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Array<Hash>] Array with all the arguments required by consuming method
-        #   including hash with all the settings required by
+        # @return [Hash] hash with all the arguments required by consuming method
+        #   including all the settings required by
         #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
         def consumption(consumer_group)
-          [
-            sanitize(
-              fetch_for(
-                :consumption,
-                consumer_group,
-                automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
-              )
+          sanitize(
+            fetch_for(
+              :consumption,
+              consumer_group,
+              automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
             )
-          ]
+          )
         end
 
         # Builds all the configuration settings for kafka consumer#subscribe method
@@ -86,17 +84,16 @@ module Karafka
         # @param topic [String] topic that we want to pause
         # @param partition [Integer] number partition that we want to pause
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Array] array with all the details required to pause kafka consumer
+        # @return [Hash] hash with all the details required to pause kafka consumer
         def pause(topic, partition, consumer_group)
-          [
-            Karafka::App.config.topic_mapper.outgoing(topic),
-            partition,
-            {
+          {
+            args: [Karafka::App.config.topic_mapper.outgoing(topic), partition],
+            kwargs: {
               timeout: consumer_group.pause_timeout,
               max_timeout: consumer_group.pause_max_timeout,
               exponential_backoff: consumer_group.pause_exponential_backoff
             }
-          ]
+          }
         end
 
         # Remaps topic details taking the topic mapper feature into consideration.

data/lib/karafka/connection/builder.rb

@@ -10,7 +10,12 @@ module Karafka
         #   to have a new Kafka client
         # @return [::Kafka::Client] returns a Kafka client
         def call(consumer_group)
-          Kafka.new(*ApiAdapter.client(consumer_group))
+          settings = ApiAdapter.client(consumer_group)
+
+          Kafka.new(
+            settings[0],
+            **settings[1]
+          )
         end
       end
     end

data/lib/karafka/connection/client.rb

@@ -33,9 +33,9 @@ module Karafka
         settings = ApiAdapter.consumption(consumer_group)
 
         if consumer_group.batch_fetching
-          kafka_consumer.each_batch(*settings) { |batch| yield(batch, :batch) }
+          kafka_consumer.each_batch(**settings) { |batch| yield(batch, :batch) }
         else
-          kafka_consumer.each_message(*settings) { |message| yield(message, :message) }
+          kafka_consumer.each_message(**settings) { |message| yield(message, :message) }
         end
       # @note We catch only the processing errors as any other are considered critical (exceptions)
       #   and should require a client restart with a backoff
@@ -64,7 +64,8 @@ module Karafka
       # @param topic [String] topic that we want to pause
       # @param partition [Integer] number partition that we want to pause
       def pause(topic, partition)
-        kafka_consumer.pause(*ApiAdapter.pause(topic, partition, consumer_group))
+        args, kwargs = ApiAdapter.pause(topic, partition, consumer_group).values_at(:args, :kwargs)
+        kafka_consumer.pause(*args, **kwargs)
       end
 
       # Marks given message as consumed
@@ -98,10 +99,12 @@ module Karafka
         # @note We don't cache the connection internally because we cache kafka_consumer that uses
         #   kafka client object instance
         @kafka_consumer ||= Builder.call(consumer_group).consumer(
-          *ApiAdapter.consumer(consumer_group)
+          **ApiAdapter.consumer(consumer_group)
         ).tap do |consumer|
           consumer_group.topics.each do |topic|
-            consumer.subscribe(*ApiAdapter.subscribe(topic))
+            settings = ApiAdapter.subscribe(topic)
+
+            consumer.subscribe(settings[0], **settings[1])
           end
         end
       rescue Kafka::ConnectionError

data/lib/karafka/instrumentation/logger.rb

@@ -20,7 +20,6 @@ module Karafka
       # @param _args Any arguments that we don't care about but that are needed in order to
       #   make this logger compatible with the default Ruby one
       def initialize(*_args)
-        ensure_dir_exists
         super(target)
         self.level = ENV_MAP[Karafka.env] || ENV_MAP['default']
       end
@@ -33,14 +32,7 @@ module Karafka
       def target
         Karafka::Helpers::MultiDelegator
           .delegate(:write, :close)
-          .to($stdout, file)
-      end
-
-      # Makes sure the log directory exists as long as we can write to it
-      def ensure_dir_exists
-        FileUtils.mkdir_p(File.dirname(log_path))
-      rescue Errno::EACCES, Errno::EROFS
-        nil
+          .to(*[$stdout, file].compact)
       end
 
       # @return [Pathname] Path to a file to which we should log
@@ -51,7 +43,11 @@ module Karafka
       # @return [File] file to which we want to write our logs
       # @note File is being opened in append mode ('a')
       def file
+        FileUtils.mkdir_p(File.dirname(log_path))
+
         @file ||= File.open(log_path, 'a')
+      rescue Errno::EACCES, Errno::EROFS
+        nil
       end
     end
   end

data/lib/karafka/instrumentation/stdout_listener.rb

@@ -103,10 +103,10 @@ module Karafka
         info "Responded from #{calling.class} using #{responder} with following data #{data}"
       end
 
-      # Logs info that we're initializing Karafka app
+      # Logs info that we're initializing Karafka framework components
       # @param _event [Dry::Events::Event] event details including payload
       def on_app_initializing(_event)
-        info "Initializing Karafka server #{::Process.pid}"
+        info "Initializing Karafka framework #{::Process.pid}"
       end
 
       # Logs info that we're running Karafka app

data/lib/karafka/server.rb

@@ -58,7 +58,7 @@ module Karafka
       def stop_supervised
         Karafka::App.stop!
 
-        # Temporary patch until https://github.com/dry-rb/dry-configurable/issues/93 is fixed
+        # See https://github.com/dry-rb/dry-configurable/issues/93
         timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
 
         # We check from time to time (for the timeout period) if all the threads finished

data/lib/karafka/templates/karafka.rb.erb

@@ -54,7 +54,7 @@ class KarafkaApp < Karafka::App
   # listen to only what you really need for given environment.
   Karafka.monitor.subscribe(WaterDrop::Instrumentation::StdoutListener.new)
   Karafka.monitor.subscribe(Karafka::Instrumentation::StdoutListener.new)
-  Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
+  # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
 
   # Uncomment that in order to achieve code reload in development mode
   # Be aware, that this might have some side-effects. Please refer to the wiki

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.4.2'
+  VERSION = '1.4.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.4.6
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -36,7 +36,7 @@ cert_chain:
   2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
   nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
   -----END CERTIFICATE-----
-date: 2021-02-26 00:00:00.000000000 Z
+date: 2021-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -122,20 +122,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '11.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '11.3'
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
@@ -313,14 +299,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.2.25
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka