karafka 1.3.0 → 1.4.0.rc1

data/lib/karafka/params/builders/batch_metadata.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    module Builders
+      # Builder for creating batch metadata object based on the batch informations
+      module BatchMetadata
+        class << self
+          # Creates metadata based on the kafka batch data
+          # @param kafka_batch [Kafka::FetchedBatch] kafka batch details
+          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
+          # @return [Karafka::Params::BatchMetadata] batch metadata object
+          def from_kafka_batch(kafka_batch, topic)
+            Karafka::Params::BatchMetadata.new(
+              batch_size: kafka_batch.messages.count,
+              first_offset: kafka_batch.first_offset,
+              highwater_mark_offset: kafka_batch.highwater_mark_offset,
+              unknown_last_offset: kafka_batch.unknown_last_offset?,
+              last_offset: kafka_batch.last_offset,
+              offset_lag: kafka_batch.offset_lag,
+              deserializer: topic.deserializer,
+              partition: kafka_batch.partition,
+              topic: topic.name
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/params.rb

@@ -12,22 +12,24 @@ module Karafka
         class << self
           # @param kafka_message [Kafka::FetchedMessage] message fetched from Kafka
           # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
-          # @return [Karafka::Params::Params] params object
+          # @return [Karafka::Params::Params] params object with payload and message metadata
           def from_kafka_message(kafka_message, topic)
-            Karafka::Params::Params
-              .new
-              .merge!(
-                'create_time' => kafka_message.create_time,
-                'headers' => kafka_message.headers || {},
-                'is_control_record' => kafka_message.is_control_record,
-                'key' => kafka_message.key,
-                'offset' => kafka_message.offset,
-                'deserializer' => topic.deserializer,
-                'partition' => kafka_message.partition,
-                'receive_time' => Time.now,
-                'topic' => kafka_message.topic,
-                'payload' => kafka_message.value
-              )
+            metadata = Karafka::Params::Metadata.new(
+              create_time: kafka_message.create_time,
+              headers: kafka_message.headers || {},
+              is_control_record: kafka_message.is_control_record,
+              key: kafka_message.key,
+              offset: kafka_message.offset,
+              deserializer: topic.deserializer,
+              partition: kafka_message.partition,
+              receive_time: Time.now,
+              topic: topic.name
+            ).freeze
+
+            Karafka::Params::Params.new(
+              kafka_message.value,
+              metadata
+            )
           end
         end
       end

data/lib/karafka/params/builders/params_batch.rb

@@ -12,11 +12,11 @@ module Karafka
           # @param topic [Karafka::Routing::Topic] topic for which we're received messages
           # @return [Karafka::Params::ParamsBatch<Karafka::Params::Params>] batch with params
           def from_kafka_messages(kafka_messages, topic)
-            params_array = kafka_messages.map! do |message|
+            params_array = kafka_messages.map do |message|
               Karafka::Params::Builders::Params.from_kafka_message(message, topic)
             end
 
-            Karafka::Params::ParamsBatch.new(params_array)
+            Karafka::Params::ParamsBatch.new(params_array).freeze
           end
         end
       end

data/lib/karafka/params/metadata.rb

@@ -2,34 +2,19 @@
 
 module Karafka
   module Params
-    # Simple metadata object that stores all non-message information received from Kafka cluster
-    # while fetching the data
-    class Metadata < Hash
-      # Attributes that should be accessible as methods as well (not only hash)
-      METHOD_ATTRIBUTES = %w[
-        batch_size
-        first_offset
-        highwater_mark_offset
-        last_offset
-        offset_lag
-        deserializer
-        partition
-        topic
-      ].freeze
-
-      private_constant :METHOD_ATTRIBUTES
-
-      METHOD_ATTRIBUTES.each do |attr|
-        # Defines a method call accessor to a particular hash field.
-        define_method(attr) do
-          self[attr]
-        end
-      end
-
-      # @return [Boolean] is the last offset known or unknown
-      def unknown_last_offset?
-        self['unknown_last_offset']
-      end
-    end
+    # Single message / params metadata details that can be accessed without the need for the
+    # payload deserialization
+    Metadata = Struct.new(
+      :create_time,
+      :headers,
+      :is_control_record,
+      :key,
+      :offset,
+      :deserializer,
+      :partition,
+      :receive_time,
+      :topic,
+      keyword_init: true
+    )
   end
 end

data/lib/karafka/params/params.rb

@@ -6,58 +6,40 @@ module Karafka
     # It provides lazy loading not only until the first usage, but also allows us to skip
     # using deserializer until we execute our logic. That way we can operate with
     # heavy-deserialization data without slowing down the whole application.
-    class Params < Hash
-      # Params attributes that should be available via a method call invocation for Kafka
-      # client compatibility.
-      # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
-      # uses those fields via method calls, so in order to be able to pass there our params
-      # objects, have to have same api.
-      METHOD_ATTRIBUTES = %w[
-        create_time
-        headers
-        is_control_record
-        key
-        offset
-        deserializer
-        deserialized
-        partition
-        receive_time
-        topic
-        payload
-      ].freeze
+    class Params
+      attr_reader :raw_payload, :metadata
 
-      private_constant :METHOD_ATTRIBUTES
-
-      METHOD_ATTRIBUTES.each do |attr|
-        # Defines a method call accessor to a particular hash field.
-        # @note Won't work for complex key names that contain spaces, etc
-        # @param key [Symbol] name of a field that we want to retrieve with a method call
-        # @example
-        #   key_attr_reader :example
-        #   params.example #=> 'my example payload'
-        define_method(attr) do
-          self[attr]
-        end
+      # @param raw_payload [Object] incoming payload before deserialization
+      # @param metadata [Karafka::Params::Metadata] message metadata object
+      def initialize(raw_payload, metadata)
+        @raw_payload = raw_payload
+        @metadata = metadata
+        @deserialized = false
+        @payload = nil
       end
 
-      # @return [Karafka::Params::Params] This method will trigger deserializer execution. If we
-      #   decide to retrieve data, deserializer will be executed to get data. Output of that will
-      #   be merged to the current object. This object will be also marked as already deserialized,
-      #   so we won't deserialize it again.
-      def deserialize!
-        return self if self['deserialized']
+      # @return [Object] lazy-deserialized data (deserialized upon first request)
+      def payload
+        return @payload if deserialized?
+
+        @payload = deserialize
+        # We mark deserialization as successful after deserialization, as in case of an error
+        # this won't be falsely set to true
+        @deserialized = true
+        @payload
+      end
 
-        self['deserialized'] = true
-        self['payload'] = deserialize
-        self
+      # @return [Boolean] did given params payload were deserialized already
+      def deserialized?
+        @deserialized
       end
 
       private
 
-      # @return [Object] deserialized data
+      # @return [Object] tries de-serializes data
       def deserialize
         Karafka.monitor.instrument('params.params.deserialize', caller: self) do
-          self['deserializer'].call(self)
+          metadata.deserializer.call(self)
         end
       rescue ::StandardError => e
         Karafka.monitor.instrument('params.params.deserialize.error', caller: self, error: e)

data/lib/karafka/params/params_batch.rb

@@ -15,47 +15,46 @@ module Karafka
         @params_array = params_array
       end
 
-      # @yieldparam [Karafka::Params::Params] each deserialized and loaded params instance
-      # @note Invocation of this method will cause loading and deserializing each param after
-      #   another. If you want to get access without deserializing, please access params_array
-      #   directly
+      # @yieldparam [Karafka::Params::Params] each params instance
+      # @note Invocation of this method will not cause loading and deserializing each param after
+      #   another.
       def each
-        @params_array.each { |param| yield(param.deserialize!) }
+        @params_array.each { |param| yield(param) }
       end
 
       # @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
       #   can be used for batch insert, etc. Without invoking all, up until first use, they won't
       #   be deserialized
       def deserialize!
-        each(&:itself)
+        each(&:payload)
       end
 
       # @return [Array<Object>] array with deserialized payloads. This method can be useful when
       #   we don't care about metadata and just want to extract all the data payloads from the
       #   batch
       def payloads
-        deserialize!.map(&:payload)
+        map(&:payload)
       end
 
-      # @return [Karafka::Params::Params] first element after the deserialization process
+      # @return [Karafka::Params::Params] first element
       def first
-        @params_array.first.deserialize!
+        @params_array.first
       end
 
-      # @return [Karafka::Params::Params] last element after the deserialization process
+      # @return [Karafka::Params::Params] last element
       def last
-        @params_array.last.deserialize!
-      end
-
-      # @return [Array<Karafka::Params::Params>] pure array with params (not deserialized)
-      def to_a
-        @params_array
+        @params_array.last
       end
 
       # @return [Integer] number of messages in the batch
       def size
         @params_array.size
       end
+
+      # @return [Array<Karafka::Params::Params>] pure array with params
+      def to_a
+        @params_array
+      end
     end
   end
 end

data/lib/karafka/serialization/json/deserializer.rb

@@ -17,8 +17,8 @@ module Karafka
         #   }
         #   Deserializer.call(params) #=> { 'a' => 1 }
         def call(params)
-          ::MultiJson.load(params['payload'])
-        rescue ::MultiJson::ParseError => e
+          params.raw_payload.nil? ? nil : ::JSON.parse(params.raw_payload)
+        rescue ::JSON::ParserError => e
           raise ::Karafka::Errors::DeserializationError, e
         end
       end

data/lib/karafka/server.rb

@@ -58,9 +58,12 @@ module Karafka
       def stop_supervised
         Karafka::App.stop!
 
+        # Temporary patch until https://github.com/dry-rb/dry-configurable/issues/93 is fixed
+        timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
+
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
-        (Karafka::App.config.shutdown_timeout * SUPERVISION_CHECK_FACTOR).to_i.times do
+        (timeout * SUPERVISION_CHECK_FACTOR).to_i.times do
           if consumer_threads.count(&:alive?).zero?
             Thread.new { Karafka.monitor.instrument('app.stopped') }.join
             return

data/lib/karafka/setup/config.rb

@@ -124,6 +124,8 @@ module Karafka
         # will be killed. Note that some Kafka operations are by definition long-running, such as
         # waiting for new messages to arrive in a partition, so don't set this value too low
         setting :socket_timeout, 30
+        # option partitioner [Object, nil] the partitioner that should be used by the client
+        setting :partitioner, nil
 
         # SSL authentication related settings
         # option ca_cert [String, nil] SSL CA certificate

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.3.0'
+  VERSION = '1.4.0.rc1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -13,30 +13,30 @@ cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
   MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
-  ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
-  NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
-  CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
-  dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
-  uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
-  MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
-  y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
-  seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
-  94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
-  TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
-  Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
-  BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
+  ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
+  NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+  CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
+  2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
+  N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
+  hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
+  sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
+  VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
+  tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
+  Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
+  LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
+  BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
   EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
-  c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
-  ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
-  xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
-  A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
-  hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
-  CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
-  i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
-  KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
-  nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+  c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
+  gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
+  2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
+  fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
+  zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
+  EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
+  65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
+  2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
+  nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
   -----END CERTIFICATE-----
-date: 2019-09-09 00:00:00.000000000 Z
+date: 2020-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -122,20 +122,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: multi_json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.12'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.12'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -156,26 +142,26 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.20'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.20'
 - !ruby/object:Gem::Dependency
@@ -184,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -208,7 +194,7 @@ dependencies:
         version: '2.1'
 description: Framework used to simplify Apache Kafka based Ruby applications development
 email:
-- maciej@coditsu.io
+- maciej@mensfeld.pl
 - pavlo.vavruk@gmail.com
 - adam99g@gmail.com
 executables:
@@ -218,14 +204,15 @@ extra_rdoc_files: []
 files:
 - ".coditsu/ci.yml"
 - ".console_irbrc"
+- ".diffend.yml"
 - ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
 - ".ruby-version"
-- ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - CONTRIBUTING.md
@@ -236,6 +223,7 @@ files:
 - bin/karafka
 - certs/mensfeld.pem
 - config/errors.yml
+- docker-compose.yml
 - karafka.gemspec
 - lib/karafka.rb
 - lib/karafka/app.rb
@@ -257,9 +245,9 @@ files:
 - lib/karafka/connection/client.rb
 - lib/karafka/connection/listener.rb
 - lib/karafka/connection/message_delegator.rb
+- lib/karafka/consumers/batch_metadata.rb
 - lib/karafka/consumers/callbacks.rb
 - lib/karafka/consumers/includer.rb
-- lib/karafka/consumers/metadata.rb
 - lib/karafka/consumers/responders.rb
 - lib/karafka/consumers/single_params.rb
 - lib/karafka/contracts.rb
@@ -278,7 +266,8 @@ files:
 - lib/karafka/instrumentation/monitor.rb
 - lib/karafka/instrumentation/proctitle_listener.rb
 - lib/karafka/instrumentation/stdout_listener.rb
-- lib/karafka/params/builders/metadata.rb
+- lib/karafka/params/batch_metadata.rb
+- lib/karafka/params/builders/batch_metadata.rb
 - lib/karafka/params/builders/params.rb
 - lib/karafka/params/builders/params_batch.rb
 - lib/karafka/params/metadata.rb
@@ -322,14 +311,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka