karafka 1.3.3 → 1.4.0.pre.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b234491eed376f4bb8a4471888c04d8ef231c71c06cbfbbfa0b73a887153fa0e
-  data.tar.gz: ee9921fc8a13766749882b7ef9e1a86ebb6e70dacf1e7a9872b8b304b7b3ac04
+  metadata.gz: 708eebd9feb79db2f96e93ce69c3f1270cb0964ecb08b1b63e6834ff144f8618
+  data.tar.gz: a6f510dc87e213b1eb8b9bd2efba1e7a65d061056c14275acb26a6895ac26a9e
 SHA512:
-  metadata.gz: 6b7310e7e15d41e2be3a1cc64452118b00fd666b9bac8fc86bd143721d09f48a08198365c1993e6ca58e13ff82e44a4c5daa115f9a3385ba9f050cab48f89c61
-  data.tar.gz: f2b50c59c770321f9f28944c2444db8bf0c48d2f2b00675fae5086e4cd5057a43fd95bda06906078d2bf6d55cb48f8b35c33f835bc6b20cc121aa342b2b63253
+  metadata.gz: 1b5667ceb4801c494b752295c18a2145c43c0aa13ca274dc55b17faec0da2996c5d50984dc3a6986885e2abb20a02aa70f8c43f16b067ac70521f51214939726
+  data.tar.gz: cc1a9dcd6e5ec3973ec490d449b5cfa1b017c0f228e76da0f08682021a8ce29860611188e4ec83e0a619ef27d2390205014733a781db792896ed62e4ecce7dc7

data/.diffend.yml

@@ -0,0 +1,3 @@
+project_id: 'de9b9933-7610-4cc4-b69b-f7e3e3c5e797'
+shareable_id: '68a8c626-b605-40ad-ac45-e3961ad7c57d'
+shareable_key: 'a3ec2dac-fba2-4b6c-b181-49e927b15057'

data/.github/workflows/ci.yml

@@ -0,0 +1,52 @@
+name: ci
+
+on:
+  push:
+  schedule:
+    - cron:  '0 1 * * *'
+
+jobs:
+  specs:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '2.7'
+          - '2.6'
+          - '2.5'
+        include:
+          - ruby: '2.7'
+            coverage: 'true'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install package dependencies
+        run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{matrix.ruby}}
+      - name: Install latest bundler
+        run: |
+          gem install bundler --no-document
+          bundle config set without 'tools benchmarks docs'
+      - name: Bundle install
+        run: |
+          bundle config set without development
+          bundle install --jobs 4 --retry 3
+      - name: Run Kafka with docker-compose
+        run: docker-compose up -d
+      - name: Run all tests
+        env:
+          GITHUB_COVERAGE: ${{matrix.coverage}}
+        run: bundle exec rspec
+  coditsu:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Run Coditsu
+        run: \curl -sSL https://api.coditsu.io/run/ci | bash

data/.ruby-version

@@ -1 +1 @@
-2.7.0
+2.7.1

data/CHANGELOG.md

@@ -1,5 +1,38 @@
 # Karafka framework changelog
 
+## 1.4.0-rc1 (unreleased)
+- Rename `Karafka::Params::Metadata` to `Karafka::Params::BatchMetadata`
+` Rename consumer `#metadata` to `#batch_metadata`
+- Separate metadata (including Karafka native metadata) from the root of params
+- Remove metadata hash dependency
+- Remove params dependency on a hash in favour of PORO
+- Remove batch metadata dependency on a hash
+- Remove MultiJson in favour of JSON in the default deserializer
+- allow accessing all the metadata without accessing the payload
+- freeze params and underlying elements except for the mutable payload
+- provide access to raw payload after serialization
+- fixes a bug where a non-deserializable (error) params would be marked as deserialized after first unsuccessful deserialization attempt
+- fixes bug where karafka would mutate internal ruby-kafka state
+- fixes bug where topic name in metadata would not be mapped using topic mappers
+- simplifies the params and params batch API, before `#payload` usage, it won't be deserialized
+- removes the `#[]` API from params to prevent from accessing raw data in a different way than #raw_payload
+- makes the params batch operations consistent as params payload is deserialized only when accessed explicitly
+
+## 1.3.7 (2020-08-11)
+- #599 - Allow metadata access without deserialization attempt (rabotyaga)
+- Sync with ruby-kafka `1.2.0` api
+
+## 1.3.6 (2020-04-24)
+- #583 - Use Karafka.logger for CLI messages (prikha)
+- #582 - Cannot only define seed brokers in consumer groups
+
+## 1.3.5 (2020-04-02)
+- #578 - ThreadError: can't be called from trap context patch
+
+## 1.3.4 (2020-02-17)
+- `dry-configurable` upgrade (solnic)
+- Remove temporary `thor` patches that are no longer needed
+
 ## 1.3.3 (2019-12-23)
 - Require `delegate` to fix missing dependency in `ruby-kafka`
 
@@ -71,6 +104,7 @@
 - #508 - Reset the consumers instances upon reconnecting to a cluster
 - [#530](https://github.com/karafka/karafka/pull/530) - expose ruby and ruby-kafka version
 - [534](https://github.com/karafka/karafka/pull/534) - Allow to use headers in the deserializer object
+- [#319](https://github.com/karafka/karafka/pull/328) - Support for exponential backoff in pause
 
 ## 1.2.11
 - [#470](https://github.com/karafka/karafka/issues/470) Karafka not working with dry-configurable 0.8
@@ -109,7 +143,6 @@
 - #318 - Partition pausing doesn't work with custom topic mappers
 - Rename ConfigAdapter to ApiAdapter to better reflect what it does
 - #317 - Manual offset committing doesn't work with custom topic mappers
-- #319 - Support for exponential backoff in pause
 
 ## 1.2.2
 - #312 - Broken for ActiveSupport 5.2.0

data/CODE_OF_CONDUCT.md

@@ -34,7 +34,7 @@ This Code of Conduct applies both within project spaces and in public spaces whe
 
 ## Enforcement
 
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maciej@coditsu.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maciej@mensfeld.pl. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
 
 Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
 

data/Gemfile

@@ -2,6 +2,8 @@
 
 source 'https://rubygems.org'
 
+plugin 'diffend'
+
 gemspec
 
 group :test do

data/Gemfile.lock

@@ -1,40 +1,41 @@
 PATH
   remote: .
   specs:
-    karafka (1.3.3)
+    karafka (1.4.0.pre.rc1)
       dry-configurable (~> 0.8)
       dry-inflector (~> 0.1)
       dry-monitor (~> 0.3)
       dry-validation (~> 1.2)
       envlogic (~> 1.1)
       irb (~> 1.0)
-      multi_json (>= 1.12)
       rake (>= 11.3)
-      ruby-kafka (>= 0.7.8)
+      ruby-kafka (>= 1.0.0)
       thor (>= 0.20)
-      waterdrop (~> 1.3.0)
+      waterdrop (~> 1.4.0)
       zeitwerk (~> 2.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.2.1)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-      zeitwerk (~> 2.2)
-    byebug (11.0.1)
-    concurrent-ruby (1.1.5)
-    delivery_boy (0.2.8)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.7)
+    delivery_boy (1.0.1)
       king_konf (~> 0.3)
-      ruby-kafka (~> 0.7.8)
-    diff-lcs (1.3)
-    digest-crc (0.4.1)
+      ruby-kafka (~> 1.0)
+    diff-lcs (1.4.4)
+    digest-crc (0.6.1)
+      rake (~> 13.0)
     docile (1.3.2)
-    dry-configurable (0.9.0)
+    dry-configurable (0.11.6)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4, >= 0.4.7)
+      dry-equalizer (~> 0.2)
     dry-container (0.7.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
@@ -46,86 +47,83 @@ GEM
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
     dry-inflector (0.2.0)
-    dry-initializer (3.0.2)
-    dry-logic (1.0.5)
+    dry-initializer (3.0.3)
+    dry-logic (1.0.7)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
-    dry-monitor (0.3.1)
+    dry-monitor (0.3.2)
       dry-configurable (~> 0.5)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-      dry-events (~> 0.1)
-    dry-schema (1.4.2)
+      dry-events (~> 0.2)
+    dry-schema (1.5.3)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.8, >= 0.8.3)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
       dry-initializer (~> 3.0)
       dry-logic (~> 1.0)
-      dry-types (~> 1.2)
-    dry-types (1.2.2)
+      dry-types (~> 1.4)
+    dry-types (1.4.0)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.3)
       dry-core (~> 0.4, >= 0.4.4)
       dry-equalizer (~> 0.3)
       dry-inflector (~> 0.1, >= 0.1.2)
       dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.4.0)
+    dry-validation (1.5.4)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.7, >= 0.7.1)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
       dry-initializer (~> 3.0)
-      dry-schema (~> 1.0, >= 1.3.1)
-    envlogic (1.1.0)
+      dry-schema (~> 1.5)
+    envlogic (1.1.2)
       dry-inflector (~> 0.1)
-    factory_bot (5.1.1)
-      activesupport (>= 4.2.0)
-    i18n (1.7.0)
+    factory_bot (6.1.0)
+      activesupport (>= 5.0.0)
+    i18n (1.8.5)
       concurrent-ruby (~> 1.0)
-    io-console (0.5.3)
-    irb (1.2.1)
+    io-console (0.5.6)
+    irb (1.2.4)
       reline (>= 0.0.1)
-    json (2.3.0)
     king_konf (0.3.7)
-    minitest (5.13.0)
-    multi_json (1.14.1)
+    minitest (5.14.1)
     rake (13.0.1)
-    reline (0.1.2)
+    reline (0.1.4)
       io-console (~> 0.5)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.0)
-      rspec-support (~> 3.9.0)
-    rspec-expectations (3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.0)
-    ruby-kafka (0.7.10)
+    rspec-support (3.9.3)
+    ruby-kafka (1.2.0)
       digest-crc
-    simplecov (0.17.1)
+    simplecov (0.19.0)
       docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.2)
     thor (1.0.1)
     thread_safe (0.3.6)
-    tzinfo (1.2.5)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
-    waterdrop (1.3.1)
-      delivery_boy (~> 0.2)
+    waterdrop (1.4.0)
+      delivery_boy (>= 0.2, < 2.x)
       dry-configurable (~> 0.8)
       dry-monitor (~> 0.3)
       dry-validation (~> 1.2)
       ruby-kafka (>= 0.7.8)
       zeitwerk (~> 2.1)
-    zeitwerk (2.2.2)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
@@ -138,4 +136,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/README.md

@@ -2,11 +2,9 @@
 
 [![Build Status](https://travis-ci.org/karafka/karafka.svg?branch=master)](https://travis-ci.org/karafka/karafka)
 
-**Note**: Documentation presented here refers to Karafka `1.3.0`.
+**Note**: Documentation presented here refers to **not yet** released Karafka `1.4.x`.
 
-If you're upgrading from `1.2.0`, please refer to our [Upgrade Notes article](https://mensfeld.pl/2019/09/karafka-framework-1-3-0-release-notes-ruby-kafka/).
-
-If you are looking for the documentation for Karafka `1.2.*`, it can be found [here](https://github.com/karafka/wiki/tree/1.2).
+If you are looking for the documentation for Karafka `1.3.x`, it can be found [here](https://github.com/karafka/wiki/tree/1.3).
 
 ## About Karafka
 
@@ -98,4 +96,4 @@ This project exists thanks to all the people who contribute.
 
 We are looking for sustainable sponsorship. If your company is relying on Karafka framework or simply want to see Karafka evolve faster to meet your requirements, please consider backing the project.
 
-Please contact [Maciej Mensfeld](mailto:maciej@coditsu.io) directly for more details.
+Please contact [Maciej Mensfeld](mailto:maciej@mensfeld.pl) directly for more details.

data/certs/mensfeld.pem

@@ -1,25 +1,25 @@
 -----BEGIN CERTIFICATE-----
 MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
-ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
-NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
-CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
-dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
-uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
-MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
-y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
-seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
-94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
-TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
-Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
-BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
+ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
+NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
+2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
+N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
+hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
+sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
+VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
+tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
+Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
+LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
+BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
 EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
-c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
-ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
-xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
-A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
-hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
-CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
-i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
-KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
-nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
+gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
+2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
+fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
+zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
+EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
+65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
+2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
+nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
 -----END CERTIFICATE-----

data/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '2'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka:1.0.1
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: localhost
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock

data/karafka.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.version     = ::Karafka::VERSION
   spec.platform    = Gem::Platform::RUBY
   spec.authors     = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
-  spec.email       = %w[maciej@coditsu.io pavlo.vavruk@gmail.com adam99g@gmail.com]
+  spec.email       = %w[maciej@mensfeld.pl pavlo.vavruk@gmail.com adam99g@gmail.com]
   spec.homepage    = 'https://github.com/karafka/karafka'
   spec.summary     = 'Ruby based framework for working with Apache Kafka'
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
@@ -23,11 +23,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'dry-validation', '~> 1.2'
   spec.add_dependency 'envlogic', '~> 1.1'
   spec.add_dependency 'irb', '~> 1.0'
-  spec.add_dependency 'multi_json', '>= 1.12'
   spec.add_dependency 'rake', '>= 11.3'
-  spec.add_dependency 'ruby-kafka', '>= 0.7.8'
+  spec.add_dependency 'ruby-kafka', '>= 1.0.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '~> 1.3.0'
+  spec.add_dependency 'waterdrop', '~> 1.4.0'
   spec.add_dependency 'zeitwerk', '~> 2.1'
 
   spec.required_ruby_version = '>= 2.5.0'

data/lib/karafka.rb

@@ -6,10 +6,10 @@
   waterdrop
   kafka
   envlogic
+  json
   thor
   forwardable
   fileutils
-  multi_json
   dry-configurable
   dry-validation
   dry/events/publisher