karafka 1.3.6 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b4de7fea7ee892f9c83db91dea9e3d16d48d423985477d3f40440bea77907906
-  data.tar.gz: ccfe4744b3bb2c19b31032e8f904716116faa50f96a9be19a4e3cdc79364ebe6
+  metadata.gz: 7bae0f63af5f741dc00d382e34a0a5a35262692c9b03930dc18fd05440bf418b
+  data.tar.gz: f9a82e8ee1324b62820f24e8dddd09fdd1ec153c50a32f4f31f50a9192983341
 SHA512:
-  metadata.gz: 74dbf97aec22f4e0f5f9a07ac9b0457c09846332c65f548d30e2b0342a2adec4b8542004aa1937012892f3374b39f83f557ef5adecacc40ae538290842165150
-  data.tar.gz: fa2072670fe7c17720e593373a4d8222163f4e789c68ec72e4e1f58f4a08688839a0c01e98115db4dfbc5c57d633b0251fe481fc9838362203961c1e4e236159
+  metadata.gz: a8fecf4be83990908c8aaf6b1e0f262391e9cdd65079362f76f68871d9ec3f45e4c0e9ceb1b277036de7f4c6ab2c140e52ee4ad855ec150c64626e9728a650a6
+  data.tar.gz: 2b626896aad5bc96bce14f0943a8d9c279dd877d2a221994f2ae56031d0bc97a33e5696b892bc8224f1c527a27e0861a79aaedd4b7716286528f6baf8f00de04

data/.diffend.yml

@@ -0,0 +1,3 @@
+project_id: 'de9b9933-7610-4cc4-b69b-f7e3e3c5e797'
+shareable_id: '68a8c626-b605-40ad-ac45-e3961ad7c57d'
+shareable_key: 'a3ec2dac-fba2-4b6c-b181-49e927b15057'

data/.github/workflows/ci.yml

@@ -0,0 +1,52 @@
+name: ci
+
+on:
+  push:
+  schedule:
+    - cron:  '0 1 * * *'
+
+jobs:
+  specs:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '2.7'
+          - '2.6'
+          - '2.5'
+        include:
+          - ruby: '2.7'
+            coverage: 'true'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install package dependencies
+        run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{matrix.ruby}}
+      - name: Install latest bundler
+        run: |
+          gem install bundler --no-document
+          bundle config set without 'tools benchmarks docs'
+      - name: Bundle install
+        run: |
+          bundle config set without development
+          bundle install --jobs 4 --retry 3
+      - name: Run Kafka with docker-compose
+        run: docker-compose up -d
+      - name: Run all tests
+        env:
+          GITHUB_COVERAGE: ${{matrix.coverage}}
+        run: bundle exec rspec
+  coditsu:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Run Coditsu
+        run: \curl -sSL https://api.coditsu.io/run/ci | bash

data/.ruby-version

@@ -1 +1 @@
-2.7.1
+2.7.2

data/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Karafka framework changelog
 
+## 1.4.1 (2020-12-04)
+- Return non-zero exit code when printing usage
+- Add support for :assignment_strategy for consumers
+
+## 1.4.0 (2020-09-05)
+- Rename `Karafka::Params::Metadata` to `Karafka::Params::BatchMetadata`
+` Rename consumer `#metadata` to `#batch_metadata`
+- Separate metadata (including Karafka native metadata) from the root of params (backwards compatibility preserved thanks to rabotyaga)
+- Remove metadata hash dependency
+- Remove params dependency on a hash in favour of PORO
+- Remove batch metadata dependency on a hash
+- Remove MultiJson in favour of JSON in the default deserializer
+- allow accessing all the metadata without accessing the payload
+- freeze params and underlying elements except for the mutable payload
+- provide access to raw payload after serialization
+- fixes a bug where a non-deserializable (error) params would be marked as deserialized after first unsuccessful deserialization attempt
+- fixes bug where karafka would mutate internal ruby-kafka state
+- fixes bug where topic name in metadata would not be mapped using topic mappers
+- simplifies the params and params batch API, before `#payload` usage, it won't be deserialized
+- removes the `#[]` API from params to prevent from accessing raw data in a different way than #raw_payload
+- makes the params batch operations consistent as params payload is deserialized only when accessed explicitly
+
+## 1.3.7 (2020-08-11)
+- #599 - Allow metadata access without deserialization attempt (rabotyaga)
+- Sync with ruby-kafka `1.2.0` api
+
 ## 1.3.6 (2020-04-24)
 - #583 - Use Karafka.logger for CLI messages (prikha)
 - #582 - Cannot only define seed brokers in consumer groups
@@ -82,6 +108,7 @@
 - #508 - Reset the consumers instances upon reconnecting to a cluster
 - [#530](https://github.com/karafka/karafka/pull/530) - expose ruby and ruby-kafka version
 - [534](https://github.com/karafka/karafka/pull/534) - Allow to use headers in the deserializer object
+- [#319](https://github.com/karafka/karafka/pull/328) - Support for exponential backoff in pause
 
 ## 1.2.11
 - [#470](https://github.com/karafka/karafka/issues/470) Karafka not working with dry-configurable 0.8
@@ -120,7 +147,6 @@
 - #318 - Partition pausing doesn't work with custom topic mappers
 - Rename ConfigAdapter to ApiAdapter to better reflect what it does
 - #317 - Manual offset committing doesn't work with custom topic mappers
-- #319 - Support for exponential backoff in pause
 
 ## 1.2.2
 - #312 - Broken for ActiveSupport 5.2.0

data/CODE_OF_CONDUCT.md

@@ -34,7 +34,7 @@ This Code of Conduct applies both within project spaces and in public spaces whe
 
 ## Enforcement
 
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maciej@coditsu.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maciej@mensfeld.pl. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
 
 Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
 

data/Gemfile

@@ -2,6 +2,8 @@
 
 source 'https://rubygems.org'
 
+plugin 'diffend'
+
 gemspec
 
 group :test do

data/Gemfile.lock

@@ -1,45 +1,45 @@
 PATH
   remote: .
   specs:
-    karafka (1.3.6)
+    karafka (1.4.1)
       dry-configurable (~> 0.8)
       dry-inflector (~> 0.1)
       dry-monitor (~> 0.3)
       dry-validation (~> 1.2)
       envlogic (~> 1.1)
       irb (~> 1.0)
-      multi_json (>= 1.12)
       rake (>= 11.3)
-      ruby-kafka (>= 0.7.8)
+      ruby-kafka (>= 1.0.0)
       thor (>= 0.20)
-      waterdrop (~> 1.3.0)
+      waterdrop (~> 1.4.0)
       zeitwerk (~> 2.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.2.2)
+    activesupport (6.0.3.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-      zeitwerk (~> 2.2)
+      zeitwerk (~> 2.2, >= 2.2.2)
     byebug (11.1.3)
-    concurrent-ruby (1.1.6)
+    concurrent-ruby (1.1.7)
     delivery_boy (1.0.1)
       king_konf (~> 0.3)
       ruby-kafka (~> 1.0)
-    diff-lcs (1.3)
-    digest-crc (0.5.1)
+    diff-lcs (1.4.4)
+    digest-crc (0.6.1)
+      rake (~> 13.0)
     docile (1.3.2)
-    dry-configurable (0.11.5)
+    dry-configurable (0.11.6)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4, >= 0.4.7)
       dry-equalizer (~> 0.2)
     dry-container (0.7.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.9)
+    dry-core (0.4.10)
       concurrent-ruby (~> 1.0)
     dry-equalizer (0.3.0)
     dry-events (0.2.0)
@@ -47,8 +47,8 @@ GEM
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
     dry-inflector (0.2.0)
-    dry-initializer (3.0.3)
-    dry-logic (1.0.6)
+    dry-initializer (3.0.4)
+    dry-logic (1.0.8)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
@@ -57,7 +57,7 @@ GEM
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
       dry-events (~> 0.2)
-    dry-schema (1.5.0)
+    dry-schema (1.5.6)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.8, >= 0.8.3)
       dry-core (~> 0.4)
@@ -72,59 +72,60 @@ GEM
       dry-equalizer (~> 0.3)
       dry-inflector (~> 0.1, >= 0.1.2)
       dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.5.0)
+    dry-validation (1.5.6)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.7, >= 0.7.1)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
       dry-initializer (~> 3.0)
-      dry-schema (~> 1.5)
+      dry-schema (~> 1.5, >= 1.5.2)
     envlogic (1.1.2)
       dry-inflector (~> 0.1)
-    factory_bot (5.1.2)
-      activesupport (>= 4.2.0)
-    i18n (1.8.2)
+    factory_bot (6.1.0)
+      activesupport (>= 5.0.0)
+    i18n (1.8.5)
       concurrent-ruby (~> 1.0)
     io-console (0.5.6)
-    irb (1.2.3)
-      reline (>= 0.0.1)
+    irb (1.2.7)
+      reline (>= 0.1.5)
     king_konf (0.3.7)
-    minitest (5.14.0)
-    multi_json (1.14.1)
+    minitest (5.14.2)
     rake (13.0.1)
-    reline (0.1.3)
+    reline (0.1.9)
       io-console (~> 0.5)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.1)
-      rspec-support (~> 3.9.1)
-    rspec-expectations (3.9.1)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.0)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.2)
-    ruby-kafka (1.0.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.0)
+    ruby-kafka (1.3.0)
       digest-crc
-    simplecov (0.18.5)
+    simplecov (0.20.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
-    simplecov-html (0.12.2)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.2)
     thor (1.0.1)
     thread_safe (0.3.6)
-    tzinfo (1.2.7)
+    tzinfo (1.2.8)
       thread_safe (~> 0.1)
-    waterdrop (1.3.4)
+    waterdrop (1.4.0)
       delivery_boy (>= 0.2, < 2.x)
       dry-configurable (~> 0.8)
       dry-monitor (~> 0.3)
       dry-validation (~> 1.2)
       ruby-kafka (>= 0.7.8)
       zeitwerk (~> 2.1)
-    zeitwerk (2.3.0)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby

data/README.md

@@ -2,11 +2,9 @@
 
 [![Build Status](https://travis-ci.org/karafka/karafka.svg?branch=master)](https://travis-ci.org/karafka/karafka)
 
-**Note**: Documentation presented here refers to Karafka `1.3.x`.
+**Note**: Documentation presented here refers to Karafka `1.4`.
 
-If you're upgrading from `1.2.0`, please refer to our [Upgrade Notes article](https://mensfeld.pl/2019/09/karafka-framework-1-3-0-release-notes-ruby-kafka/).
-
-If you are looking for the documentation for Karafka `1.2.x`, it can be found [here](https://github.com/karafka/wiki/tree/1.2).
+If you are looking for the documentation for Karafka `1.3.x`, it can be found [here](https://github.com/karafka/wiki/tree/1.3).
 
 ## About Karafka
 
@@ -98,4 +96,4 @@ This project exists thanks to all the people who contribute.
 
 We are looking for sustainable sponsorship. If your company is relying on Karafka framework or simply want to see Karafka evolve faster to meet your requirements, please consider backing the project.
 
-Please contact [Maciej Mensfeld](mailto:maciej@coditsu.io) directly for more details.
+Please contact [Maciej Mensfeld](mailto:maciej@mensfeld.pl) directly for more details.

data/certs/mensfeld.pem

@@ -1,25 +1,25 @@
 -----BEGIN CERTIFICATE-----
 MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
-ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
-NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
-CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
-dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
-uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
-MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
-y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
-seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
-94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
-TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
-Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
-BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
+ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
+NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
+2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
+N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
+hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
+sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
+VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
+tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
+Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
+LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
+BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
 EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
-c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
-ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
-xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
-A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
-hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
-CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
-i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
-KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
-nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
+gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
+2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
+fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
+zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
+EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
+65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
+2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
+nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
 -----END CERTIFICATE-----

data/config/errors.yml

@@ -37,3 +37,5 @@ en:
         Unknown consumer group
       does_not_exist:
         Given file does not exist or cannot be read
+      does_not_respond_to_call: >
+        needs to respond to a #call method

data/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '2'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka:1.0.1
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: localhost
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock

data/karafka.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.version     = ::Karafka::VERSION
   spec.platform    = Gem::Platform::RUBY
   spec.authors     = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
-  spec.email       = %w[maciej@coditsu.io pavlo.vavruk@gmail.com adam99g@gmail.com]
+  spec.email       = %w[maciej@mensfeld.pl pavlo.vavruk@gmail.com adam99g@gmail.com]
   spec.homepage    = 'https://github.com/karafka/karafka'
   spec.summary     = 'Ruby based framework for working with Apache Kafka'
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
@@ -23,11 +23,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'dry-validation', '~> 1.2'
   spec.add_dependency 'envlogic', '~> 1.1'
   spec.add_dependency 'irb', '~> 1.0'
-  spec.add_dependency 'multi_json', '>= 1.12'
   spec.add_dependency 'rake', '>= 11.3'
-  spec.add_dependency 'ruby-kafka', '>= 0.7.8'
+  spec.add_dependency 'ruby-kafka', '>= 1.0.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '~> 1.3.0'
+  spec.add_dependency 'waterdrop', '~> 1.4.0'
   spec.add_dependency 'zeitwerk', '~> 2.1'
 
   spec.required_ruby_version = '>= 2.5.0'

data/lib/karafka.rb

@@ -6,10 +6,10 @@
   waterdrop
   kafka
   envlogic
+  json
   thor
   forwardable
   fileutils
-  multi_json
   dry-configurable
   dry-validation
   dry/events/publisher

data/lib/karafka/assignment_strategies/round_robin.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Strategies for Kafka partitions assignments
+  module AssignmentStrategies
+    # Standard RoundRobin strategy
+    class RoundRobin < SimpleDelegator
+      def initialize
+        super(Kafka::RoundRobinAssignmentStrategy.new)
+      end
+    end
+  end
+end