karafka 1.2.13 → 1.3.4

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.2.13'
+  VERSION = '1.3.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.2.13
+  version: 1.3.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -9,23 +9,35 @@ authors:
 - Adam Gwozdowski
 autorequire: 
 bindir: bin
-cert_chain: []
-date: 2019-05-19 00:00:00.000000000 Z
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
+  ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
+  NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+  CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
+  dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
+  uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
+  MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
+  y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
+  seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
+  94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
+  TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
+  Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
+  BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
+  EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
+  c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
+  ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
+  xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
+  A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
+  hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
+  CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
+  i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
+  KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
+  nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+  -----END CERTIFICATE-----
+date: 2020-02-17 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '4.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: dry-configurable
   requirement: !ruby/object:Gem::Requirement
@@ -74,16 +86,30 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: envlogic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: irb
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -125,61 +151,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '11.3'
 - !ruby/object:Gem::Dependency
-  name: require_all
+  name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 0.7.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 0.7.8
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.20'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.20'
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: 1.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
-  name: waterdrop
+  name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.4
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.4
+        version: '2.1'
 description: Framework used to simplify Apache Kafka based Ruby applications development
 email:
 - maciej@coditsu.io
@@ -190,8 +216,11 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".coditsu.yml"
+- ".coditsu/ci.yml"
 - ".console_irbrc"
+- ".github/FUNDING.yml"
+- ".github/ISSUE_TEMPLATE/bug_report.md"
+- ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
@@ -205,6 +234,7 @@ files:
 - MIT-LICENCE
 - README.md
 - bin/karafka
+- certs/mensfeld.pem
 - config/errors.yml
 - karafka.gemspec
 - lib/karafka.rb
@@ -213,9 +243,6 @@ files:
 - lib/karafka/backends/inline.rb
 - lib/karafka/base_consumer.rb
 - lib/karafka/base_responder.rb
-- lib/karafka/callbacks.rb
-- lib/karafka/callbacks/config.rb
-- lib/karafka/callbacks/dsl.rb
 - lib/karafka/cli.rb
 - lib/karafka/cli/base.rb
 - lib/karafka/cli/console.rb
@@ -223,32 +250,44 @@ files:
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
+- lib/karafka/code_reloader.rb
 - lib/karafka/connection/api_adapter.rb
+- lib/karafka/connection/batch_delegator.rb
 - lib/karafka/connection/builder.rb
 - lib/karafka/connection/client.rb
-- lib/karafka/connection/delegator.rb
 - lib/karafka/connection/listener.rb
+- lib/karafka/connection/message_delegator.rb
 - lib/karafka/consumers/callbacks.rb
 - lib/karafka/consumers/includer.rb
+- lib/karafka/consumers/metadata.rb
 - lib/karafka/consumers/responders.rb
 - lib/karafka/consumers/single_params.rb
+- lib/karafka/contracts.rb
+- lib/karafka/contracts/config.rb
+- lib/karafka/contracts/consumer_group.rb
+- lib/karafka/contracts/consumer_group_topic.rb
+- lib/karafka/contracts/responder_usage.rb
+- lib/karafka/contracts/server_cli_options.rb
 - lib/karafka/errors.rb
 - lib/karafka/fetcher.rb
 - lib/karafka/helpers/class_matcher.rb
 - lib/karafka/helpers/config_retriever.rb
+- lib/karafka/helpers/inflector.rb
 - lib/karafka/helpers/multi_delegator.rb
-- lib/karafka/instrumentation/listener.rb
 - lib/karafka/instrumentation/logger.rb
 - lib/karafka/instrumentation/monitor.rb
-- lib/karafka/loader.rb
-- lib/karafka/params/dsl.rb
+- lib/karafka/instrumentation/proctitle_listener.rb
+- lib/karafka/instrumentation/stdout_listener.rb
+- lib/karafka/params/builders/metadata.rb
+- lib/karafka/params/builders/params.rb
+- lib/karafka/params/builders/params_batch.rb
+- lib/karafka/params/metadata.rb
+- lib/karafka/params/params.rb
 - lib/karafka/params/params_batch.rb
-- lib/karafka/parsers/json.rb
-- lib/karafka/patches/dry_configurable.rb
 - lib/karafka/patches/ruby_kafka.rb
 - lib/karafka/persistence/client.rb
-- lib/karafka/persistence/consumer.rb
-- lib/karafka/persistence/topic.rb
+- lib/karafka/persistence/consumers.rb
+- lib/karafka/persistence/topics.rb
 - lib/karafka/process.rb
 - lib/karafka/responders/builder.rb
 - lib/karafka/responders/topic.rb
@@ -259,29 +298,23 @@ files:
 - lib/karafka/routing/router.rb
 - lib/karafka/routing/topic.rb
 - lib/karafka/routing/topic_mapper.rb
-- lib/karafka/schemas/config.rb
-- lib/karafka/schemas/consumer_group.rb
-- lib/karafka/schemas/consumer_group_topic.rb
-- lib/karafka/schemas/responder_usage.rb
-- lib/karafka/schemas/server_cli_options.rb
+- lib/karafka/serialization/json/deserializer.rb
+- lib/karafka/serialization/json/serializer.rb
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb
-- lib/karafka/setup/configurators/base.rb
-- lib/karafka/setup/configurators/params.rb
 - lib/karafka/setup/configurators/water_drop.rb
 - lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb
-- lib/karafka/templates/application_consumer.rb.example
-- lib/karafka/templates/application_responder.rb.example
-- lib/karafka/templates/karafka.rb.example
+- lib/karafka/templates/application_consumer.rb.erb
+- lib/karafka/templates/application_responder.rb.erb
+- lib/karafka/templates/karafka.rb.erb
 - lib/karafka/version.rb
 - log/.gitkeep
 homepage: https://github.com/karafka/karafka
 licenses:
 - MIT
 metadata: {}
-post_install_message: "\e[93mWarning:\e[0m If you're using Kafka 0.10, please lock
-  ruby-kafka in your Gemfile to version '0.6.8':\ngem 'ruby-kafka', '~> 0.6.8'\n"
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -289,14 +322,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka

data/lib/karafka/callbacks.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Additional callbacks that are used to trigger some things in given places during the
-  # system lifecycle
-  # @note Those callbacks aren't the same as consumer callbacks as they are not related to the
-  #   lifecycle of particular messages fetches but rather to the internal flow process.
-  #   They cannot be defined on a consumer callback level because for some of those,
-  #   there aren't consumers in the memory yet and/or they aren't per consumer thread
-  module Callbacks
-    # Types of system callbacks that we have that are not related to consumers
-    TYPES = %i[
-      after_init
-      before_fetch_loop
-    ].freeze
-
-    class << self
-      TYPES.each do |callback_type|
-        # Executes given callbacks set at a given moment with provided arguments
-        define_method callback_type do |*args|
-          Karafka::App
-            .config
-            .callbacks
-            .send(callback_type)
-            .each { |callback| callback.call(*args) }
-        end
-      end
-    end
-  end
-end

data/lib/karafka/callbacks/config.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Callbacks
-    # Additional configuration required to store procs that we will execute upon callback trigger
-    module Config
-      # Builds up internal callback accumulators
-      # @param klass [Class] Class that we extend with callback config
-      def self.extended(klass)
-        # option internal [Hash] - optional - internal karafka configuration settings that should
-        #   never be changed by users directly
-        klass.setting :callbacks do
-          Callbacks::TYPES.each do |callback_type|
-            # option [Array<Proc>] an array of blocks that will be executed at a given moment
-            #   depending on the callback type
-            setting callback_type, []
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/callbacks/dsl.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Callbacks
-    # App level dsl to define callbacks
-    module Dsl
-      Callbacks::TYPES.each do |callback_type|
-        # Allows us to define a block, that will be executed for a given moment
-        # @param [Block] block that should be executed after the initialization process
-        define_method callback_type do |&block|
-          config.callbacks.send(callback_type).push block
-        end
-      end
-    end
-  end
-end

data/lib/karafka/connection/delegator.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Class that delegates processing of messages for which we listen to a proper processor
-    module Delegator
-      class << self
-        # Delegates messages (does something with them)
-        # It will either schedule or run a proper processor action for messages
-        # @note This should be looped to obtain a constant delegating of new messages
-        # @note We catch all the errors here, to make sure that none failures
-        #   for a given consumption will affect other consumed messages
-        #   If we wouldn't catch it, it would propagate up until killing the thread
-        # @note It is a one huge method, because of performance reasons. It is much faster then
-        #   using send or invoking additional methods
-        # @param group_id [String] group_id of a group from which a given message came
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
-        def call(group_id, kafka_messages)
-          # @note We always get messages by topic and partition so we can take topic from the
-          # first one and it will be valid for all the messages
-          topic = Persistence::Topic.fetch(group_id, kafka_messages[0].topic)
-          consumer = Persistence::Consumer.fetch(topic, kafka_messages[0].partition)
-
-          Karafka.monitor.instrument(
-            'connection.delegator.call',
-            caller: self,
-            consumer: consumer,
-            kafka_messages: kafka_messages
-          ) do
-            # Depending on a case (persisted or not) we might use new consumer instance per
-            # each batch, or use the same one for all of them (for implementing buffering, etc.)
-            if topic.batch_consuming
-              consumer.params_batch = kafka_messages
-              consumer.call
-            else
-              kafka_messages.each do |kafka_message|
-                consumer.params_batch = [kafka_message]
-                consumer.call
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/instrumentation/listener.rb

@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Instrumentation
-    # Default listener that hooks up to our instrumentation and uses its events for logging
-    # It can be removed/replaced or anything without any harm to the Karafka app flow
-    module Listener
-      # Log levels that we use in this particular listener
-      USED_LOG_LEVELS = %i[
-        debug
-        info
-        error
-        fatal
-      ].freeze
-
-      # Injects WaterDrop listener logger actions
-      extend WaterDrop::Instrumentation::Listener
-
-      class << self
-        # Logs details about incoming messages and with which consumer we will consume them
-        # @param event [Dry::Events::Event] event details including payload
-        def on_connection_delegator_call(event)
-          consumer = event[:consumer]
-          topic = consumer.topic.name
-          kafka_messages = event[:kafka_messages]
-          info "#{kafka_messages.count} messages on #{topic} topic delegated to #{consumer.class}"
-        end
-
-        # Logs details about each received message value parsing
-        # @param event [Dry::Events::Event] event details including payload
-        def on_params_params_parse(event)
-          # Keep in mind, that a caller here is a param object not a controller,
-          # so it returns a topic as a string, not a routing topic
-          debug "Params parsing for #{event[:caller].topic} topic successful in #{event[:time]} ms"
-        end
-
-        # Logs unsuccessful parsing attempts of incoming data
-        # @param event [Dry::Events::Event] event details including payload
-        def on_params_params_parse_error(event)
-          error "Params parsing error for #{event[:caller].topic} topic: #{event[:error]}"
-        end
-
-        # Logs errors that occured in a listener fetch loop
-        # @param event [Dry::Events::Event] event details including payload
-        # @note It's an error as we can recover from it not a fatal
-        def on_connection_listener_fetch_loop_error(event)
-          error "Listener fetch loop error: #{event[:error]}"
-        end
-
-        # Logs errors that are related to the connection itself
-        # @note Karafka will attempt to reconnect, so an error not a fatal
-        # @param event [Dry::Events::Event] event details including payload
-        def on_connection_client_fetch_loop_error(event)
-          error "Client fetch loop error: #{event[:error]}"
-        end
-
-        # Logs info about crashed fetcher
-        # @note If this happens, Karafka will shutdown as it means a critical error
-        #   in one of the threads
-        # @param event [Dry::Events::Event] event details including payload
-        def on_fetcher_call_error(event)
-          fatal "Fetcher crash due to an error: #{event[:error]}"
-        end
-
-        # Logs info about processing of a certain dataset with an inline backend
-        # @param event [Dry::Events::Event] event details including payload
-        def on_backends_inline_process(event)
-          count = event[:caller].send(:params_batch).to_a.size
-          topic = event[:caller].topic.name
-          time = event[:time]
-          info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
-        end
-
-        # Logs info about system signals that Karafka received
-        # @param event [Dry::Events::Event] event details including payload
-        def on_process_notice_signal(event)
-          info "Received #{event[:signal]} system signal"
-        end
-
-        # Logs info about responder usage withing a controller flow
-        # @param event [Dry::Events::Event] event details including payload
-        def on_consumers_responders_respond_with(event)
-          calling = event[:caller].class
-          responder = calling.topic.responder
-          data = event[:data]
-          info "Responded from #{calling} using #{responder} with following data #{data}"
-        end
-
-        # Logs info that we're going to stop the Karafka server
-        # @param _event [Dry::Events::Event] event details including payload
-        def on_server_stop(_event)
-          # We use a separate thread as logging can't be called from trap context
-          Thread.new { info "Stopping Karafka server #{::Process.pid}" }
-        end
-
-        # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
-        #   forced exit
-        # @param _event [Dry::Events::Event] event details including payload
-        def on_server_stop_error(_event)
-          # We use a separate thread as logging can't be called from trap context
-          Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
-        end
-
-        USED_LOG_LEVELS.each do |log_level|
-          define_method log_level do |*args|
-            Karafka.logger.send(log_level, *args)
-          end
-        end
-      end
-    end
-  end
-end