karafka 1.2.12 → 1.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5935dd3f3ddfc3d592bcdff859e63b4cee55b5b12ed3cdd49fdcc399aa32c47
-  data.tar.gz: 1d4eee47f902cd65ebbe4025b52d799d58467f5102bc595f2f5d37fd0bbb4713
+  metadata.gz: b234491eed376f4bb8a4471888c04d8ef231c71c06cbfbbfa0b73a887153fa0e
+  data.tar.gz: ee9921fc8a13766749882b7ef9e1a86ebb6e70dacf1e7a9872b8b304b7b3ac04
 SHA512:
-  metadata.gz: 8cdbbe2cf8bc790f6923f4cfc92902774a14d614f08f0f34e649bec6de1dae5444c4477957089de77e57328d48d85d2ac626a63ecf652aefbbf35977397ce2cc
-  data.tar.gz: 3c15a030cb9f5cf2ce5cfc5d36213c637546e279803b4c0ff80565b9c4587e3172f8517a8c8fbd50ffe3867f9df3d4f43a582c00a0271ef1f39b2b8d0aa9141f
+  metadata.gz: 6b7310e7e15d41e2be3a1cc64452118b00fd666b9bac8fc86bd143721d09f48a08198365c1993e6ca58e13ff82e44a4c5daa115f9a3385ba9f050cab48f89c61
+  data.tar.gz: f2b50c59c770321f9f28944c2444db8bf0c48d2f2b00675fae5086e4cd5057a43fd95bda06906078d2bf6d55cb48f8b35c33f835bc6b20cc121aa342b2b63253

data/{.coditsu.yml → .coditsu/ci.yml}

@@ -1,3 +1,3 @@
+repository_id: 'd4482d42-f6b5-44ba-a5e4-00989ac519ee'
 api_key: <%= ENV['CODITSU_API_KEY'] %>
 api_secret: <%= ENV['CODITSU_API_SECRET'] %>
-repository_id: <%= ENV['CODITSU_REPOSITORY_ID'] %>

data/.console_irbrc

@@ -1,11 +1,9 @@
 # irbrc for Karafka console
-require 'karafka'
-require Karafka.boot_file
 
 IRB.conf[:AUTO_INDENT] = true
 IRB.conf[:SAVE_HISTORY] = 1000
 IRB.conf[:USE_READLINE] = true
-IRB.conf[:HISTORY_FILE] = "#{Karafka::App.root}/.irb-history"
+IRB.conf[:HISTORY_FILE] = ".irb-history"
 IRB.conf[:LOAD_MODULES] = [] unless IRB.conf.key?(:LOAD_MODULES)
 
 unless IRB.conf[:LOAD_MODULES].include?('irb/completion')

data/.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+open_collective: karafka

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,50 @@
+---
+name: Bug Report
+about: Report an issue with Karafka you've discovered.
+---
+
+*Be clear, concise and precise in your description of the problem.
+Open an issue with a descriptive title and a summary in grammatically correct,
+complete sentences.*
+
+*Use the template below when reporting bugs. Please, make sure that
+you're running the latest stable Karafka and that the problem you're reporting
+hasn't been reported (and potentially fixed) already.*
+
+*Before filing the ticket you should replace all text above the horizontal
+rule with your own words.*
+
+--------
+
+## Expected behavior
+
+Describe here how you expected Karafka to behave in this particular situation.
+
+## Actual behavior
+
+Describe here what actually happened.
+
+## Steps to reproduce the problem
+
+This is extremely important! Providing us with a reliable way to reproduce
+a problem will expedite its solution.
+
+## Your setup details
+
+Please provide kafka version and the output of `karafka info` or `bundle exec karafka info` if using Bundler.
+
+Here's an example:
+
+```
+$ [bundle exec] karafka info
+Karafka version: 1.3.0
+Ruby version: 2.6.3
+Ruby-kafka version: 0.7.9
+Application client id: karafka-local
+Backend: inline
+Batch fetching: true
+Batch consuming: true
+Boot file: /app/karafka/karafka.rb
+Environment: development
+Kafka seed brokers: ["kafka://kafka:9092"]
+```

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature Request
+about: Suggest new Karafka features or improvements to existing features.
+---
+
+## Is your feature request related to a problem? Please describe.
+
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+## Describe the solution you'd like
+
+A clear and concise description of what you want to happen.
+
+## Describe alternatives you've considered
+
+A clear and concise description of any alternative solutions or features you've considered.
+
+## Additional context
+
+Add any other context or screenshots about the feature request here.

data/.gitignore

@@ -6,6 +6,7 @@
 app.god
 
 # minimal Rails specific artifacts
+/.coditsu/local.yml
 db/*.sqlite3
 /log/development.log
 /log/production.log

data/.ruby-version

@@ -1 +1 @@
-2.6.1
+2.7.0

data/.travis.yml

@@ -12,38 +12,25 @@ test: &test
   stage: Test
   language: ruby
   before_install:
-    - gem install bundler
-    - gem update --system
+    - yes | gem update --system
   script: bundle exec rspec
 
 jobs:
   include:
     - <<: *test
-      rvm: 2.6.1
+      rvm: 2.7.0
     - <<: *test
-      rvm: 2.5.3
+      rvm: 2.6.5
     - <<: *test
-      rvm: 2.4.5
-    - <<: *test
-      rvm: 2.3.8
+      rvm: 2.5.7
 
     - stage: coditsu
       language: ruby
-      rvm: 2.6.1
+      rvm: 2.7.0
       before_install:
-        - gem update --system
-        - gem install bundler
-      before_script:
-        - docker create -v /sources --name sources alpine:3.4 /bin/true
-        - docker cp ./ sources:/sources
-      script: >
-        docker run
-        -e CODITSU_API_KEY
-        -e CODITSU_API_SECRET
-        -e CODITSU_REPOSITORY_ID
-        --volumes-from sources
-        coditsu/build-runner:latest
+        - yes | gem update --system
+      script: \curl -sSL https://api.coditsu.io/run/ci | bash
 
 stages:
-  - coditsu
   - test
+  - coditsu

data/CHANGELOG.md

@@ -1,7 +1,76 @@
 # Karafka framework changelog
 
-## 1.2.12
+## 1.3.3 (2019-12-23)
+- Require `delegate` to fix missing dependency in `ruby-kafka`
+
+## 1.3.2 (2019-12-23)
+- #561 - Allow `thor` 1.0.x usage in Karafka
+- #567 - Ruby 2.7.0 support + unfreeze of a frozen string fix
+
+## 1.3.1 (2019-11-11)
+- #545 - Makes sure the log directory exists when is possible (robertomiranda)
+- Ruby 2.6.5 support
+- #551 - add support for DSA keys
+- #549 - Missing directories after `karafka install` (nijikon)
+
+## 1.3.0 (2019-09-09)
+- Drop support for Ruby 2.4
+- YARD docs tags cleanup
+
+## 1.3.0.rc1 (2019-07-31)
+- Drop support for Kafka 0.10 in favor of native support for Kafka 0.11.
+- Update ruby-kafka to the 0.7 version
+- Support messages headers receiving
+- Message bus unification
+- Parser available in metadata
+- Cleanup towards moving to a non-global state app management
+- Drop Ruby 2.3 support
+- Support for Ruby 2.6.3
+- `Karafka::Loader` has been removed in favor of Zeitwerk
+- Schemas are now contracts
+- #393 - Reorganize responders - removed `multiple_usage` constrain
+- #388 - ssl_client_cert_chain sync
+- #300 - Store value in a value key and replace its content with parsed version - without root merge
+- #331 - Disallow building groups without topics
+- #340 - Instrumentation unification. Better and more consistent naming
+- #340 - Procline instrumentation for a nicer process name
+- #342 - Change default for `fetcher_max_queue_size` from `100` to `10` to lower max memory usage
+- #345 - Cleanup exceptions names
+- #341 - Split connection delegator into batch delegator and single_delegator
+- #351 - Rename `#retrieve!` to `#parse!` on params and `#parsed` to `parse!` on params batch.
+- #351 - Adds '#first' for params_batch that returns parsed first element from the params_batch object.
+- #360 - Single params consuming mode automatically parses data specs
+- #359 - Divide mark_as_consumed into mark_as_consumed and mark_as_consumed!
+- #356 - Provide a `#values` for params_batch to extract only values of objects from the params_batch
+- #363 - Too shallow ruby-kafka version lock
+- #354 - Expose consumer heartbeat
+- #377 - Remove the persistent setup in favor of persistence
+- #375 - Sidekiq Backend parser mismatch
+- #369 - Single consumer can support more than one topic
+- #288 - Drop dependency on `activesupport` gem
+- #371 - SASL over SSL
+- #392 - Move params redundant data to metadata
+- #335 - Metadata access from within the consumer
+- #402 - Delayed reconnection upon critical failures
+- #405 - `reconnect_timeout` value is now being validated
+- #437 - Specs ensuring that the `#437` won't occur in the `1.3` release
+- #426 - ssl client cert key password
+- #444 - add certificate and private key validation
+- #460 - Decouple responder "parser" (generator?) from topic.parser (benissimo)
+- #463 - Split parsers into serializers / deserializers
+- #473 - Support SASL OAuthBearer Authentication
+- #475 - Disallow subscribing to the same topic with multiple consumers
+- #485 - Setting shutdown_timeout to nil kills the app without waiting for anything
+- #487 - Make listeners as instances
 - #29 - Consumer class names must have the word "Consumer" in it in order to work (Sidekiq backend)
+- #491 - irb is missing for console to work
+- #502 - Karafka process hangs when sending multiple sigkills
+- #506 - ssl_verify_hostname sync
+- #483 - Upgrade dry-validation before releasing 1.3
+- #492 - Use Zeitwerk for code reload in development
+- #508 - Reset the consumers instances upon reconnecting to a cluster
+- [#530](https://github.com/karafka/karafka/pull/530) - expose ruby and ruby-kafka version
+- [534](https://github.com/karafka/karafka/pull/534) - Allow to use headers in the deserializer object
 
 ## 1.2.11
 - [#470](https://github.com/karafka/karafka/issues/470) Karafka not working with dry-configurable 0.8
@@ -24,6 +93,7 @@
 ## 1.2.6
 - Lock WaterDrop to 1.2.3
 - Lock Ruby-Kafka to 0.6.x (support for 0.7 will be added in Karafka 1.3)
+- #382 - Full logging with AR, etc for development mode when there is Rails integration
 
 ## 1.2.5
 - #354 - Expose consumer heartbeat
@@ -34,10 +104,12 @@
 
 ## 1.2.3
 - #313 - support PLAINTEXT and SSL for scheme
-- #320 - Pausing indefinetely with nil pause timeout doesn't work
+- #288 - drop activesupport callbacks in favor of notifications
+- #320 - Pausing indefinitely with nil pause timeout doesn't work
 - #318 - Partition pausing doesn't work with custom topic mappers
 - Rename ConfigAdapter to ApiAdapter to better reflect what it does
 - #317 - Manual offset committing doesn't work with custom topic mappers
+- #319 - Support for exponential backoff in pause
 
 ## 1.2.2
 - #312 - Broken for ActiveSupport 5.2.0
@@ -162,7 +234,7 @@
 - Switch to multi json so everyone can use their favourite JSON parser
 - Added jruby support in general and in Travis
 - #196 - Topic mapper does not map topics when subscribing thanks to @webandtech
-- #96 - Karafka server - possiblity to run it only for a certain topics
+- #96 - Karafka server - possibility to run it only for a certain topics
 - ~~karafka worker cli option is removed (please use sidekiq directly)~~ - restored, bad idea
 - (optional) pausing upon processing failures ```pause_timeout```
 - Karafka console main process no longer intercepts irb errors
@@ -170,7 +242,7 @@
 - #204 - Long running controllers
 - Better internal API to handle multiple usage cases using ```Karafka::Controllers::Includer```
 - #207 - Rename before_enqueued to after_received
-- #147 - Deattach Karafka from Sidekiq by extracting Sidekiq backend
+- #147 - De-attach Karafka from Sidekiq by extracting Sidekiq backend
 
 ### New features and improvements
 
@@ -255,7 +327,7 @@
 - Waterdrop 0.3.2.1 with kafka.hosts instead of kafka_hosts
 - #105 - Karafka::Monitor#caller_label not working with inherited monitors
 - #99 - Standalone mode (without Sidekiq)
-- #97 - Buffer responders single topics before send (prevalidation)
+- #97 - Buffer responders single topics before send (pre-validation)
 - Better control over consumer thanks to additional config options
 - #111 - Dynamic worker assignment based on the income params
 - Long shutdown time fix
@@ -263,7 +335,7 @@
 ## 0.5.0
 - Removed Zookeeper totally as dependency
 - Better group and partition rebalancing
-- Automatic thread management (no need for tunning) - each topic is a separate actor/thread
+- Automatic thread management (no need for tuning) - each topic is a separate actor/thread
 - Moved from Poseidon into Ruby-Kafka
 - No more max_concurrency setting
 - After you define your App class and routes (and everything else) you need to add execute App.boot!
@@ -279,14 +351,14 @@
 - Ruby 2.2.* support dropped
 - Using App name as a Kafka client_id
 - Automatic Capistrano integration
-- Responders support for handling better responses pipelining and better responses flow description and design (see README for more details)
+- Responders support for handling better responses pipe-lining and better responses flow description and design (see README for more details)
 - Gem bump
 - Readme updates
 - karafka flow CLI command for printing the application flow
-- Some internal refactorings
+- Some internal refactoring
 
 ## 0.4.2
-- #87 - Reconsume mode with crone for better Rails/Rack integration
+- #87 - Re-consume mode with crone for better Rails/Rack integration
 - Moved Karafka server related stuff into separate Karafka::Server class
 - Renamed Karafka::Runner into Karafka::Fetcher
 - Gem bump
@@ -298,7 +370,7 @@
 
 ## 0.4.1
 - Explicit throw(:abort) required to halt before_enqueue (like in Rails 5)
-- #61 - Autodiscover Kafka brokers based on Zookeeper data
+- #61 - autodiscovery of Kafka brokers based on Zookeeper data
 - #63 - Graceful shutdown with current offset state during data processing
 - #65 - Example of NewRelic monitor is outdated
 - #71 - Setup should be executed after user code is loaded
@@ -354,7 +426,7 @@
 - Added Karafka::Monitoring that allows to add custom logging and monitoring with external libraries and systems
 - Moved logging functionality into Karafka::Monitoring default monitoring
 - Added possibility to provide own monitoring as long as in responds to #notice and #notice_error
-- Standarized logging format for all logs
+- Standardized logging format for all logs
 
 ## 0.3.0
 - Switched from custom ParserError for each parser to general catching of Karafka::Errors::ParseError and its descendants
@@ -371,7 +443,7 @@
 
 ## 0.1.19
 - Internal call - schedule naming change
-- Enqueue to perform_async naming in controller to follow Sidekiqs naming convention
+- Enqueue to perform_async naming in controller to follow Sidekiq naming convention
 - Gem bump
 
 ## 0.1.18
@@ -382,7 +454,7 @@
 - Changed Karafka::Connection::Cluster tp Karafka::Connection::ActorCluster to distinguish between a single thread actor cluster for multiple topic connection and a future feature that will allow process clusterization.
 - Add an ability to use user-defined parsers for a messages
 - Lazy load params for before callbacks
-- Automatic loading/initializng all workers classes during startup (so Sidekiq won't fail with unknown workers exception)
+- Automatic loading/initializing all workers classes during startup (so Sidekiq won't fail with unknown workers exception)
 - Params are now private to controller
 - Added bootstrap method to app.rb
 
@@ -423,7 +495,7 @@
 - Added worker logger
 
 ## 0.1.8
-- Droped local env suppot in favour of [Envlogic](https://github.com/karafka/envlogic) - no changes in API
+- Dropped local env support in favour of [Envlogic](https://github.com/karafka/envlogic) - no changes in API
 
 ## 0.1.7
 - Karafka option for Redis hosts (not localhost only)
@@ -453,7 +525,7 @@
 
 ## 0.1.1
 - README updates
-- Raketasks updates
+- Rake tasks updates
 - Rake installation task
 - Changelog file added
 

data/CONTRIBUTING.md

@@ -4,7 +4,7 @@
 
 First, thank you for considering contributing to karafka! It's people like you that make the open source community such a great community! 😊
 
-We welcome any type of contribution, not only code. You can help with 
+We welcome any type of contribution, not only code. You can help with:
 - **QA**: file bug reports, the more details you can give the better (e.g. screenshots with the console open)
 - **Marketing**: writing blog posts, howto's, printing stickers, ...
 - **Community**: presenting the project at meetups, organizing a dedicated meetup for the local community, ...

data/Gemfile

@@ -4,12 +4,9 @@ source 'https://rubygems.org'
 
 gemspec
 
-group :development, :test do
-  gem 'benchmark-ips'
-end
-
 group :test do
+  gem 'byebug'
+  gem 'factory_bot'
   gem 'rspec'
   gem 'simplecov'
-  gem 'timecop'
 end

data/Gemfile.lock

@@ -1,126 +1,141 @@
 PATH
   remote: .
   specs:
-    karafka (1.2.12)
-      activesupport (>= 4.0)
+    karafka (1.3.3)
       dry-configurable (~> 0.8)
       dry-inflector (~> 0.1)
       dry-monitor (~> 0.3)
-      dry-validation (~> 0.11)
-      envlogic (~> 1.0)
+      dry-validation (~> 1.2)
+      envlogic (~> 1.1)
+      irb (~> 1.0)
       multi_json (>= 1.12)
       rake (>= 11.3)
-      require_all (>= 1.4)
-      ruby-kafka (>= 0.6)
-      thor (~> 0.20)
-      waterdrop (~> 1.2.4)
+      ruby-kafka (>= 0.7.8)
+      thor (>= 0.20)
+      waterdrop (~> 1.3.0)
+      zeitwerk (~> 2.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.3)
+    activesupport (6.0.2.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    benchmark-ips (2.7.2)
+      zeitwerk (~> 2.2)
+    byebug (11.0.1)
     concurrent-ruby (1.1.5)
-    delivery_boy (0.2.7)
-      king_konf (~> 0.2)
-      ruby-kafka (~> 0.5)
+    delivery_boy (0.2.8)
+      king_konf (~> 0.3)
+      ruby-kafka (~> 0.7.8)
     diff-lcs (1.3)
     digest-crc (0.4.1)
-    docile (1.3.1)
-    dry-configurable (0.8.2)
+    docile (1.3.2)
+    dry-configurable (0.9.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4, >= 0.4.7)
-    dry-container (0.7.0)
+    dry-container (0.7.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.7)
+    dry-core (0.4.9)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.2.2)
-    dry-events (0.1.1)
+    dry-equalizer (0.3.0)
+    dry-events (0.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-    dry-inflector (0.1.2)
-    dry-logic (0.6.1)
+    dry-inflector (0.2.0)
+    dry-initializer (3.0.2)
+    dry-logic (1.0.5)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
-    dry-monitor (0.3.0)
+    dry-monitor (0.3.1)
       dry-configurable (~> 0.5)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
       dry-events (~> 0.1)
-    dry-types (0.14.1)
+    dry-schema (1.4.2)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.8, >= 0.8.3)
+      dry-core (~> 0.4)
+      dry-equalizer (~> 0.2)
+      dry-initializer (~> 3.0)
+      dry-logic (~> 1.0)
+      dry-types (~> 1.2)
+    dry-types (1.2.2)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.3)
       dry-core (~> 0.4, >= 0.4.4)
-      dry-equalizer (~> 0.2)
+      dry-equalizer (~> 0.3)
       dry-inflector (~> 0.1, >= 0.1.2)
-      dry-logic (~> 0.5, >= 0.5)
-    dry-validation (0.13.1)
+      dry-logic (~> 1.0, >= 1.0.2)
+    dry-validation (1.4.0)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1, >= 0.1.3)
-      dry-core (~> 0.2, >= 0.2.1)
+      dry-container (~> 0.7, >= 0.7.1)
+      dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-      dry-logic (~> 0.5, >= 0.5.0)
-      dry-types (~> 0.14.0)
+      dry-initializer (~> 3.0)
+      dry-schema (~> 1.0, >= 1.3.1)
     envlogic (1.1.0)
       dry-inflector (~> 0.1)
-    i18n (1.6.0)
+    factory_bot (5.1.1)
+      activesupport (>= 4.2.0)
+    i18n (1.7.0)
       concurrent-ruby (~> 1.0)
-    json (2.1.0)
+    io-console (0.5.3)
+    irb (1.2.1)
+      reline (>= 0.0.1)
+    json (2.3.0)
     king_konf (0.3.7)
-    minitest (5.11.3)
-    multi_json (1.13.1)
-    null-logger (0.1.5)
-    rake (12.3.2)
-    require_all (2.0.0)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.2)
+    minitest (5.13.0)
+    multi_json (1.14.1)
+    rake (13.0.1)
+    reline (0.1.2)
+      io-console (~> 0.5)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.0)
+      rspec-support (~> 3.9.0)
+    rspec-expectations (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
-    ruby-kafka (0.7.6)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.0)
+    ruby-kafka (0.7.10)
       digest-crc
-    simplecov (0.16.1)
+    simplecov (0.17.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    thor (0.20.3)
+    thor (1.0.1)
     thread_safe (0.3.6)
-    timecop (0.9.1)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    waterdrop (1.2.5)
+    waterdrop (1.3.1)
       delivery_boy (~> 0.2)
       dry-configurable (~> 0.8)
       dry-monitor (~> 0.3)
-      dry-validation (~> 0.11)
-      null-logger (~> 0.1)
-      ruby-kafka (>= 0.6)
+      dry-validation (~> 1.2)
+      ruby-kafka (>= 0.7.8)
+      zeitwerk (~> 2.1)
+    zeitwerk (2.2.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  benchmark-ips
+  byebug
+  factory_bot
   karafka!
   rspec
   simplecov
-  timecop
 
 BUNDLED WITH
-   2.0.1
+   2.1.2