karafka 1.3.0 → 1.4.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffe10d1ca48b0b218191231ec6969f3a767f5a1a4010dca0ef00fa44c2eaab67
-  data.tar.gz: 27d1b52ba3782b562176b7a6de73563bdd48c250bc0576a1821c55bfcec07bff
+  metadata.gz: e1bc3a30364a87750723e5c338cdddffb6c6c4c2965e12a7379fbe96a6e7756c
+  data.tar.gz: 073fe6b2ffe1d7cbd53670330cd14780b48dda4f364a4ba160a42c13a1914c2d
 SHA512:
-  metadata.gz: 85bdedfe0791c7d17abbc54ed49df83e2c9cbfb048124107d403559e72b2884909bcc5974c98a738744f80184315b2ee31cc8bd1048cc5a10230e6b6f0184e3d
-  data.tar.gz: 382074bee041ed27776571816a982fbd7f0af6b6f9cfa8fa393113acf1c2d4cbfb0a7573d2bcd16f2662667d1b8175546e281deb5a7b66ce30b5d9c4c19d005d
+  metadata.gz: d7bbc41af65bf2590d7e045e552b10121f202b7a031681c7ac5d20148e8c6c053eced4aa98ece0661a1ead6df1c163b8d03e06b19ca5f83346c4575ec10489c7
+  data.tar.gz: 522f1a73478633269333f04ebdfffb988e26a405b659e44bf44da5757b105dd24bc16f3214415d4258c325c1c4fa899f54ef322a335b31b859dda131a70e2021

checksums.yaml.gz.sig

@@ -1,2 +1,2 @@
-���k�W%g
-�9�RHj��ia/�ۋ#���O��#�K��rS�����d��!q�:�v��}��.��Th�-S�j�T$�.�3�C+�
+R�ظ#o��B�d���Us��ˠl�˧�Π,��!�V��a��5��1i*�~��/��Ս��`����ͨ�;*p�<R�SDN�����e�ҥ5'|7�ho*�;VԦm�,!��l��I���-��$��� �Jy��'�ߗ%��!_^��6$`(M�3}��{�1,��K]|�E%�!�\�P�N�r)�����?n%ҷȅ�����c+/?�5>�w�|Ic5L%N��,?�xN��������4h�
+(��|&&���]7'�@
�)!��XM���N��e��������u�Wm�h������

data/.diffend.yml

@@ -0,0 +1,3 @@
+project_id: 'de9b9933-7610-4cc4-b69b-f7e3e3c5e797'
+shareable_id: '68a8c626-b605-40ad-ac45-e3961ad7c57d'
+shareable_key: 'a3ec2dac-fba2-4b6c-b181-49e927b15057'

data/.github/workflows/ci.yml

@@ -0,0 +1,52 @@
+name: ci
+
+on:
+  push:
+  schedule:
+    - cron:  '0 1 * * *'
+
+jobs:
+  specs:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '2.7'
+          - '2.6'
+          - '2.5'
+        include:
+          - ruby: '2.7'
+            coverage: 'true'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install package dependencies
+        run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{matrix.ruby}}
+      - name: Install latest bundler
+        run: |
+          gem install bundler --no-document
+          bundle config set without 'tools benchmarks docs'
+      - name: Bundle install
+        run: |
+          bundle config set without development
+          bundle install --jobs 4 --retry 3
+      - name: Run Kafka with docker-compose
+        run: docker-compose up -d
+      - name: Run all tests
+        env:
+          GITHUB_COVERAGE: ${{matrix.coverage}}
+        run: bundle exec rspec
+  coditsu:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Run Coditsu
+        run: \curl -sSL https://api.coditsu.io/run/ci | bash

data/.ruby-version

@@ -1 +1 @@
-2.6.3
+2.7.1

data/CHANGELOG.md

@@ -1,5 +1,51 @@
 # Karafka framework changelog
 
+## 1.4.0.rc1 (2020-08-25)
+- Rename `Karafka::Params::Metadata` to `Karafka::Params::BatchMetadata`
+` Rename consumer `#metadata` to `#batch_metadata`
+- Separate metadata (including Karafka native metadata) from the root of params
+- Remove metadata hash dependency
+- Remove params dependency on a hash in favour of PORO
+- Remove batch metadata dependency on a hash
+- Remove MultiJson in favour of JSON in the default deserializer
+- allow accessing all the metadata without accessing the payload
+- freeze params and underlying elements except for the mutable payload
+- provide access to raw payload after serialization
+- fixes a bug where a non-deserializable (error) params would be marked as deserialized after first unsuccessful deserialization attempt
+- fixes bug where karafka would mutate internal ruby-kafka state
+- fixes bug where topic name in metadata would not be mapped using topic mappers
+- simplifies the params and params batch API, before `#payload` usage, it won't be deserialized
+- removes the `#[]` API from params to prevent from accessing raw data in a different way than #raw_payload
+- makes the params batch operations consistent as params payload is deserialized only when accessed explicitly
+
+## 1.3.7 (2020-08-11)
+- #599 - Allow metadata access without deserialization attempt (rabotyaga)
+- Sync with ruby-kafka `1.2.0` api
+
+## 1.3.6 (2020-04-24)
+- #583 - Use Karafka.logger for CLI messages (prikha)
+- #582 - Cannot only define seed brokers in consumer groups
+
+## 1.3.5 (2020-04-02)
+- #578 - ThreadError: can't be called from trap context patch
+
+## 1.3.4 (2020-02-17)
+- `dry-configurable` upgrade (solnic)
+- Remove temporary `thor` patches that are no longer needed
+
+## 1.3.3 (2019-12-23)
+- Require `delegate` to fix missing dependency in `ruby-kafka`
+
+## 1.3.2 (2019-12-23)
+- #561 - Allow `thor` 1.0.x usage in Karafka
+- #567 - Ruby 2.7.0 support + unfreeze of a frozen string fix
+
+## 1.3.1 (2019-11-11)
+- #545 - Makes sure the log directory exists when is possible (robertomiranda)
+- Ruby 2.6.5 support
+- #551 - add support for DSA keys
+- #549 - Missing directories after `karafka install` (nijikon)
+
 ## 1.3.0 (2019-09-09)
 - Drop support for Ruby 2.4
 - YARD docs tags cleanup
@@ -58,6 +104,7 @@
 - #508 - Reset the consumers instances upon reconnecting to a cluster
 - [#530](https://github.com/karafka/karafka/pull/530) - expose ruby and ruby-kafka version
 - [534](https://github.com/karafka/karafka/pull/534) - Allow to use headers in the deserializer object
+- [#319](https://github.com/karafka/karafka/pull/328) - Support for exponential backoff in pause
 
 ## 1.2.11
 - [#470](https://github.com/karafka/karafka/issues/470) Karafka not working with dry-configurable 0.8
@@ -92,11 +139,10 @@
 ## 1.2.3
 - #313 - support PLAINTEXT and SSL for scheme
 - #288 - drop activesupport callbacks in favor of notifications
-- #320 - Pausing indefinetely with nil pause timeout doesn't work
+- #320 - Pausing indefinitely with nil pause timeout doesn't work
 - #318 - Partition pausing doesn't work with custom topic mappers
 - Rename ConfigAdapter to ApiAdapter to better reflect what it does
 - #317 - Manual offset committing doesn't work with custom topic mappers
-- #319 - Support for exponential backoff in pause
 
 ## 1.2.2
 - #312 - Broken for ActiveSupport 5.2.0
@@ -221,7 +267,7 @@
 - Switch to multi json so everyone can use their favourite JSON parser
 - Added jruby support in general and in Travis
 - #196 - Topic mapper does not map topics when subscribing thanks to @webandtech
-- #96 - Karafka server - possiblity to run it only for a certain topics
+- #96 - Karafka server - possibility to run it only for a certain topics
 - ~~karafka worker cli option is removed (please use sidekiq directly)~~ - restored, bad idea
 - (optional) pausing upon processing failures ```pause_timeout```
 - Karafka console main process no longer intercepts irb errors
@@ -229,7 +275,7 @@
 - #204 - Long running controllers
 - Better internal API to handle multiple usage cases using ```Karafka::Controllers::Includer```
 - #207 - Rename before_enqueued to after_received
-- #147 - Deattach Karafka from Sidekiq by extracting Sidekiq backend
+- #147 - De-attach Karafka from Sidekiq by extracting Sidekiq backend
 
 ### New features and improvements
 
@@ -314,7 +360,7 @@
 - Waterdrop 0.3.2.1 with kafka.hosts instead of kafka_hosts
 - #105 - Karafka::Monitor#caller_label not working with inherited monitors
 - #99 - Standalone mode (without Sidekiq)
-- #97 - Buffer responders single topics before send (prevalidation)
+- #97 - Buffer responders single topics before send (pre-validation)
 - Better control over consumer thanks to additional config options
 - #111 - Dynamic worker assignment based on the income params
 - Long shutdown time fix
@@ -322,7 +368,7 @@
 ## 0.5.0
 - Removed Zookeeper totally as dependency
 - Better group and partition rebalancing
-- Automatic thread management (no need for tunning) - each topic is a separate actor/thread
+- Automatic thread management (no need for tuning) - each topic is a separate actor/thread
 - Moved from Poseidon into Ruby-Kafka
 - No more max_concurrency setting
 - After you define your App class and routes (and everything else) you need to add execute App.boot!
@@ -338,14 +384,14 @@
 - Ruby 2.2.* support dropped
 - Using App name as a Kafka client_id
 - Automatic Capistrano integration
-- Responders support for handling better responses pipelining and better responses flow description and design (see README for more details)
+- Responders support for handling better responses pipe-lining and better responses flow description and design (see README for more details)
 - Gem bump
 - Readme updates
 - karafka flow CLI command for printing the application flow
-- Some internal refactorings
+- Some internal refactoring
 
 ## 0.4.2
-- #87 - Reconsume mode with crone for better Rails/Rack integration
+- #87 - Re-consume mode with crone for better Rails/Rack integration
 - Moved Karafka server related stuff into separate Karafka::Server class
 - Renamed Karafka::Runner into Karafka::Fetcher
 - Gem bump
@@ -357,7 +403,7 @@
 
 ## 0.4.1
 - Explicit throw(:abort) required to halt before_enqueue (like in Rails 5)
-- #61 - Autodiscover Kafka brokers based on Zookeeper data
+- #61 - autodiscovery of Kafka brokers based on Zookeeper data
 - #63 - Graceful shutdown with current offset state during data processing
 - #65 - Example of NewRelic monitor is outdated
 - #71 - Setup should be executed after user code is loaded
@@ -413,7 +459,7 @@
 - Added Karafka::Monitoring that allows to add custom logging and monitoring with external libraries and systems
 - Moved logging functionality into Karafka::Monitoring default monitoring
 - Added possibility to provide own monitoring as long as in responds to #notice and #notice_error
-- Standarized logging format for all logs
+- Standardized logging format for all logs
 
 ## 0.3.0
 - Switched from custom ParserError for each parser to general catching of Karafka::Errors::ParseError and its descendants
@@ -430,7 +476,7 @@
 
 ## 0.1.19
 - Internal call - schedule naming change
-- Enqueue to perform_async naming in controller to follow Sidekiqs naming convention
+- Enqueue to perform_async naming in controller to follow Sidekiq naming convention
 - Gem bump
 
 ## 0.1.18
@@ -441,7 +487,7 @@
 - Changed Karafka::Connection::Cluster tp Karafka::Connection::ActorCluster to distinguish between a single thread actor cluster for multiple topic connection and a future feature that will allow process clusterization.
 - Add an ability to use user-defined parsers for a messages
 - Lazy load params for before callbacks
-- Automatic loading/initializng all workers classes during startup (so Sidekiq won't fail with unknown workers exception)
+- Automatic loading/initializing all workers classes during startup (so Sidekiq won't fail with unknown workers exception)
 - Params are now private to controller
 - Added bootstrap method to app.rb
 
@@ -482,7 +528,7 @@
 - Added worker logger
 
 ## 0.1.8
-- Droped local env suppot in favour of [Envlogic](https://github.com/karafka/envlogic) - no changes in API
+- Dropped local env support in favour of [Envlogic](https://github.com/karafka/envlogic) - no changes in API
 
 ## 0.1.7
 - Karafka option for Redis hosts (not localhost only)
@@ -512,7 +558,7 @@
 
 ## 0.1.1
 - README updates
-- Raketasks updates
+- Rake tasks updates
 - Rake installation task
 - Changelog file added
 

data/CODE_OF_CONDUCT.md

@@ -34,7 +34,7 @@ This Code of Conduct applies both within project spaces and in public spaces whe
 
 ## Enforcement
 
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maciej@coditsu.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maciej@mensfeld.pl. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
 
 Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
 

data/Gemfile

@@ -2,6 +2,8 @@
 
 source 'https://rubygems.org'
 
+plugin 'diffend'
+
 gemspec
 
 group :test do

data/Gemfile.lock

@@ -1,127 +1,129 @@
 PATH
   remote: .
   specs:
-    karafka (1.3.0)
+    karafka (1.4.0.rc1)
       dry-configurable (~> 0.8)
       dry-inflector (~> 0.1)
       dry-monitor (~> 0.3)
       dry-validation (~> 1.2)
       envlogic (~> 1.1)
       irb (~> 1.0)
-      multi_json (>= 1.12)
       rake (>= 11.3)
-      ruby-kafka (>= 0.7.8)
-      thor (~> 0.20)
-      waterdrop (~> 1.3.0)
+      ruby-kafka (>= 1.0.0)
+      thor (>= 0.20)
+      waterdrop (~> 1.4.0)
       zeitwerk (~> 2.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.0)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-      zeitwerk (~> 2.1, >= 2.1.8)
-    byebug (11.0.1)
-    concurrent-ruby (1.1.5)
-    delivery_boy (0.2.8)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.7)
+    delivery_boy (1.0.1)
       king_konf (~> 0.3)
-      ruby-kafka (~> 0.7.8)
-    diff-lcs (1.3)
-    digest-crc (0.4.1)
+      ruby-kafka (~> 1.0)
+    diff-lcs (1.4.4)
+    digest-crc (0.6.1)
+      rake (~> 13.0)
     docile (1.3.2)
-    dry-configurable (0.8.3)
+    dry-configurable (0.11.6)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4, >= 0.4.7)
+      dry-equalizer (~> 0.2)
     dry-container (0.7.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
     dry-core (0.4.9)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.2.2)
+    dry-equalizer (0.3.0)
     dry-events (0.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-    dry-inflector (0.1.2)
-    dry-initializer (3.0.1)
-    dry-logic (1.0.3)
+    dry-inflector (0.2.0)
+    dry-initializer (3.0.3)
+    dry-logic (1.0.7)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
-    dry-monitor (0.3.1)
+    dry-monitor (0.3.2)
       dry-configurable (~> 0.5)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-      dry-events (~> 0.1)
-    dry-schema (1.3.3)
+      dry-events (~> 0.2)
+    dry-schema (1.5.3)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.8, >= 0.8.3)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
       dry-initializer (~> 3.0)
       dry-logic (~> 1.0)
-      dry-types (~> 1.0)
-    dry-types (1.1.1)
+      dry-types (~> 1.4)
+    dry-types (1.4.0)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.3)
       dry-core (~> 0.4, >= 0.4.4)
-      dry-equalizer (~> 0.2, >= 0.2.2)
+      dry-equalizer (~> 0.3)
       dry-inflector (~> 0.1, >= 0.1.2)
       dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.3.1)
+    dry-validation (1.5.4)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.7, >= 0.7.1)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
       dry-initializer (~> 3.0)
-      dry-schema (~> 1.0, >= 1.3.1)
-    envlogic (1.1.0)
+      dry-schema (~> 1.5)
+    envlogic (1.1.2)
       dry-inflector (~> 0.1)
-    factory_bot (5.0.2)
-      activesupport (>= 4.2.0)
-    i18n (1.6.0)
+    factory_bot (6.1.0)
+      activesupport (>= 5.0.0)
+    i18n (1.8.5)
       concurrent-ruby (~> 1.0)
-    irb (1.0.0)
-    json (2.2.0)
+    io-console (0.5.6)
+    irb (1.2.4)
+      reline (>= 0.0.1)
     king_konf (0.3.7)
-    minitest (5.11.3)
-    multi_json (1.13.1)
-    rake (12.3.3)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.2)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.4)
+    minitest (5.14.1)
+    rake (13.0.1)
+    reline (0.1.4)
+      io-console (~> 0.5)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.1)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.2)
-    ruby-kafka (0.7.10)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    ruby-kafka (1.2.0)
       digest-crc
-    simplecov (0.17.0)
+    simplecov (0.19.0)
       docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
-    thor (0.20.3)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.2)
+    thor (1.0.1)
     thread_safe (0.3.6)
-    tzinfo (1.2.5)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
-    waterdrop (1.3.0)
-      delivery_boy (~> 0.2)
+    waterdrop (1.4.0)
+      delivery_boy (>= 0.2, < 2.x)
       dry-configurable (~> 0.8)
       dry-monitor (~> 0.3)
       dry-validation (~> 1.2)
       ruby-kafka (>= 0.7.8)
       zeitwerk (~> 2.1)
-    zeitwerk (2.1.10)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
@@ -134,4 +136,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.0.2
+   2.1.4