karafka 1.3.3 → 1.4.0.pre.rc1

data/lib/karafka/params/params_batch.rb

@@ -15,47 +15,46 @@ module Karafka
         @params_array = params_array
       end
 
-      # @yieldparam [Karafka::Params::Params] each deserialized and loaded params instance
-      # @note Invocation of this method will cause loading and deserializing each param after
-      #   another. If you want to get access without deserializing, please access params_array
-      #   directly
+      # @yieldparam [Karafka::Params::Params] each params instance
+      # @note Invocation of this method will not cause loading and deserializing each param after
+      #   another.
       def each
-        @params_array.each { |param| yield(param.deserialize!) }
+        @params_array.each { |param| yield(param) }
       end
 
       # @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
       #   can be used for batch insert, etc. Without invoking all, up until first use, they won't
       #   be deserialized
       def deserialize!
-        each(&:itself)
+        each(&:payload)
       end
 
       # @return [Array<Object>] array with deserialized payloads. This method can be useful when
       #   we don't care about metadata and just want to extract all the data payloads from the
       #   batch
       def payloads
-        deserialize!.map(&:payload)
+        map(&:payload)
       end
 
-      # @return [Karafka::Params::Params] first element after the deserialization process
+      # @return [Karafka::Params::Params] first element
       def first
-        @params_array.first.deserialize!
+        @params_array.first
       end
 
-      # @return [Karafka::Params::Params] last element after the deserialization process
+      # @return [Karafka::Params::Params] last element
       def last
-        @params_array.last.deserialize!
-      end
-
-      # @return [Array<Karafka::Params::Params>] pure array with params (not deserialized)
-      def to_a
-        @params_array
+        @params_array.last
       end
 
       # @return [Integer] number of messages in the batch
       def size
         @params_array.size
       end
+
+      # @return [Array<Karafka::Params::Params>] pure array with params
+      def to_a
+        @params_array
+      end
     end
   end
 end

data/lib/karafka/serialization/json/deserializer.rb

@@ -17,8 +17,8 @@ module Karafka
         #   }
         #   Deserializer.call(params) #=> { 'a' => 1 }
         def call(params)
-          ::MultiJson.load(params['payload'])
-        rescue ::MultiJson::ParseError => e
+          params.raw_payload.nil? ? nil : ::JSON.parse(params.raw_payload)
+        rescue ::JSON::ParserError => e
           raise ::Karafka::Errors::DeserializationError, e
         end
       end

data/lib/karafka/server.rb

@@ -58,9 +58,12 @@ module Karafka
       def stop_supervised
         Karafka::App.stop!
 
+        # Temporary patch until https://github.com/dry-rb/dry-configurable/issues/93 is fixed
+        timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
+
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
-        (Karafka::App.config.shutdown_timeout * SUPERVISION_CHECK_FACTOR).to_i.times do
+        (timeout * SUPERVISION_CHECK_FACTOR).to_i.times do
           if consumer_threads.count(&:alive?).zero?
             Thread.new { Karafka.monitor.instrument('app.stopped') }.join
             return

data/lib/karafka/setup/config.rb

@@ -124,6 +124,8 @@ module Karafka
         # will be killed. Note that some Kafka operations are by definition long-running, such as
         # waiting for new messages to arrive in a partition, so don't set this value too low
         setting :socket_timeout, 30
+        # option partitioner [Object, nil] the partitioner that should be used by the client
+        setting :partitioner, nil
 
         # SSL authentication related settings
         # option ca_cert [String, nil] SSL CA certificate

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.3.3'
+  VERSION = '1.4.0-rc1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.4.0.pre.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -13,30 +13,30 @@ cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
   MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
-  ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
-  NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
-  CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
-  dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
-  uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
-  MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
-  y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
-  seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
-  94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
-  TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
-  Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
-  BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
+  ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
+  NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+  CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
+  2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
+  N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
+  hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
+  sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
+  VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
+  tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
+  Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
+  LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
+  BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
   EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
-  c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
-  ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
-  xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
-  A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
-  hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
-  CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
-  i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
-  KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
-  nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+  c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
+  gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
+  2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
+  fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
+  zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
+  EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
+  65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
+  2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
+  nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
   -----END CERTIFICATE-----
-date: 2019-12-26 00:00:00.000000000 Z
+date: 2020-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -122,20 +122,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: multi_json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.12'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.12'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -156,14 +142,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -184,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -208,7 +194,7 @@ dependencies:
         version: '2.1'
 description: Framework used to simplify Apache Kafka based Ruby applications development
 email:
-- maciej@coditsu.io
+- maciej@mensfeld.pl
 - pavlo.vavruk@gmail.com
 - adam99g@gmail.com
 executables:
@@ -218,14 +204,15 @@ extra_rdoc_files: []
 files:
 - ".coditsu/ci.yml"
 - ".console_irbrc"
+- ".diffend.yml"
 - ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
 - ".ruby-version"
-- ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - CONTRIBUTING.md
@@ -236,6 +223,7 @@ files:
 - bin/karafka
 - certs/mensfeld.pem
 - config/errors.yml
+- docker-compose.yml
 - karafka.gemspec
 - lib/karafka.rb
 - lib/karafka/app.rb
@@ -257,9 +245,9 @@ files:
 - lib/karafka/connection/client.rb
 - lib/karafka/connection/listener.rb
 - lib/karafka/connection/message_delegator.rb
+- lib/karafka/consumers/batch_metadata.rb
 - lib/karafka/consumers/callbacks.rb
 - lib/karafka/consumers/includer.rb
-- lib/karafka/consumers/metadata.rb
 - lib/karafka/consumers/responders.rb
 - lib/karafka/consumers/single_params.rb
 - lib/karafka/contracts.rb
@@ -278,7 +266,8 @@ files:
 - lib/karafka/instrumentation/monitor.rb
 - lib/karafka/instrumentation/proctitle_listener.rb
 - lib/karafka/instrumentation/stdout_listener.rb
-- lib/karafka/params/builders/metadata.rb
+- lib/karafka/params/batch_metadata.rb
+- lib/karafka/params/builders/batch_metadata.rb
 - lib/karafka/params/builders/params.rb
 - lib/karafka/params/builders/params_batch.rb
 - lib/karafka/params/metadata.rb
@@ -325,11 +314,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka

data/.travis.yml

@@ -1,36 +0,0 @@
-services:
-  - docker
-
-dist: trusty
-sudo: false
-cache: bundler
-
-git:
-  depth: false
-
-test: &test
-  stage: Test
-  language: ruby
-  before_install:
-    - yes | gem update --system
-  script: bundle exec rspec
-
-jobs:
-  include:
-    - <<: *test
-      rvm: 2.7.0
-    - <<: *test
-      rvm: 2.6.5
-    - <<: *test
-      rvm: 2.5.7
-
-    - stage: coditsu
-      language: ruby
-      rvm: 2.7.0
-      before_install:
-        - yes | gem update --system
-      script: \curl -sSL https://api.coditsu.io/run/ci | bash
-
-stages:
-  - test
-  - coditsu

data/lib/karafka/consumers/metadata.rb

@@ -1,10 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Consumers
-    # Brings the metadata into consumers that support batch_fetching
-    module Metadata
-      attr_accessor :metadata
-    end
-  end
-end

data/lib/karafka/params/builders/metadata.rb

@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    module Builders
-      # Builder for creating metadata object based on the message or batch informations
-      # @note We have 2 ways of creating metadata based on the way ruby-kafka operates
-      module Metadata
-        class << self
-          # Creates metadata based on the kafka batch data
-          # @param kafka_batch [Kafka::FetchedBatch] kafka batch details
-          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
-          # @return [Karafka::Params::Metadata] metadata object
-          def from_kafka_batch(kafka_batch, topic)
-            Karafka::Params::Metadata
-              .new
-              .merge!(
-                'batch_size' => kafka_batch.messages.count,
-                'first_offset' => kafka_batch.first_offset,
-                'highwater_mark_offset' => kafka_batch.highwater_mark_offset,
-                'last_offset' => kafka_batch.last_offset,
-                'offset_lag' => kafka_batch.offset_lag,
-                'deserializer' => topic.deserializer,
-                'partition' => kafka_batch.partition,
-                'topic' => kafka_batch.topic,
-                'unknown_last_offset' => kafka_batch.unknown_last_offset?
-              )
-          end
-        end
-      end
-    end
-  end
-end