karafka 1.1.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 26ee2938706de62609da9faf4179b3071fdc9e62
-  data.tar.gz: eedc149b1ade99a21d82f32ad3ec9009b065cc89
+SHA256:
+  metadata.gz: ffe10d1ca48b0b218191231ec6969f3a767f5a1a4010dca0ef00fa44c2eaab67
+  data.tar.gz: 27d1b52ba3782b562176b7a6de73563bdd48c250bc0576a1821c55bfcec07bff
 SHA512:
-  metadata.gz: 2e25b2453563d9330955248307eee910147899fefb15bc197313f79012c132f0970fede7f0a0fd50b6966267c9d7d48f82f50b75fea927baa8c01bf68ebee305
-  data.tar.gz: 902fd68ad24eb5400976d9f6b3cc5d955617a70e0b14186e2b27e8292fcff2d59aa013231834ad679c9694eebd6809589cf98715151d89a1637d6e71e2f5a14e
+  metadata.gz: 85bdedfe0791c7d17abbc54ed49df83e2c9cbfb048124107d403559e72b2884909bcc5974c98a738744f80184315b2ee31cc8bd1048cc5a10230e6b6f0184e3d
+  data.tar.gz: 382074bee041ed27776571816a982fbd7f0af6b6f9cfa8fa393113acf1c2d4cbfb0a7573d2bcd16f2662667d1b8175546e281deb5a7b66ce30b5d9c4c19d005d

checksums.yaml.gz.sig

@@ -0,0 +1,2 @@
+���k�W%g
+�9�RHj��ia/�ۋ#���O��#�K��rS�����d��!q�:�v��}��.��Th�-S�j�T$�.�3�C+�

data/.coditsu/ci.yml

@@ -0,0 +1,3 @@
+repository_id: 'd4482d42-f6b5-44ba-a5e4-00989ac519ee'
+api_key: <%= ENV['CODITSU_API_KEY'] %>
+api_secret: <%= ENV['CODITSU_API_SECRET'] %>

data/.console_irbrc

@@ -1,11 +1,9 @@
 # irbrc for Karafka console
-require 'karafka'
-require Karafka.boot_file
 
 IRB.conf[:AUTO_INDENT] = true
 IRB.conf[:SAVE_HISTORY] = 1000
 IRB.conf[:USE_READLINE] = true
-IRB.conf[:HISTORY_FILE] = "#{Karafka::App.root}/.irb-history"
+IRB.conf[:HISTORY_FILE] = ".irb-history"
 IRB.conf[:LOAD_MODULES] = [] unless IRB.conf.key?(:LOAD_MODULES)
 
 unless IRB.conf[:LOAD_MODULES].include?('irb/completion')

data/.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+open_collective: karafka

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,50 @@
+---
+name: Bug Report
+about: Report an issue with Karafka you've discovered.
+---
+
+*Be clear, concise and precise in your description of the problem.
+Open an issue with a descriptive title and a summary in grammatically correct,
+complete sentences.*
+
+*Use the template below when reporting bugs. Please, make sure that
+you're running the latest stable Karafka and that the problem you're reporting
+hasn't been reported (and potentially fixed) already.*
+
+*Before filing the ticket you should replace all text above the horizontal
+rule with your own words.*
+
+--------
+
+## Expected behavior
+
+Describe here how you expected Karafka to behave in this particular situation.
+
+## Actual behavior
+
+Describe here what actually happened.
+
+## Steps to reproduce the problem
+
+This is extremely important! Providing us with a reliable way to reproduce
+a problem will expedite its solution.
+
+## Your setup details
+
+Please provide kafka version and the output of `karafka info` or `bundle exec karafka info` if using Bundler.
+
+Here's an example:
+
+```
+$ [bundle exec] karafka info
+Karafka version: 1.3.0
+Ruby version: 2.6.3
+Ruby-kafka version: 0.7.9
+Application client id: karafka-local
+Backend: inline
+Batch fetching: true
+Batch consuming: true
+Boot file: /app/karafka/karafka.rb
+Environment: development
+Kafka seed brokers: ["kafka://kafka:9092"]
+```

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature Request
+about: Suggest new Karafka features or improvements to existing features.
+---
+
+## Is your feature request related to a problem? Please describe.
+
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+## Describe the solution you'd like
+
+A clear and concise description of what you want to happen.
+
+## Describe alternatives you've considered
+
+A clear and concise description of any alternative solutions or features you've considered.
+
+## Additional context
+
+Add any other context or screenshots about the feature request here.

data/.gitignore

@@ -6,6 +6,7 @@
 app.god
 
 # minimal Rails specific artifacts
+/.coditsu/local.yml
 db/*.sqlite3
 /log/development.log
 /log/production.log

data/.ruby-version

@@ -1 +1 @@
-2.4.2
+2.6.3

data/.travis.yml

@@ -1,17 +1,36 @@
-language: ruby
+services:
+  - docker
+
+dist: trusty
 sudo: false
-rvm:
-  - 2.3.0
-  - 2.3.1
-  - 2.3.2
-  - 2.3.3
-  - 2.3.4
-  - 2.4.0
-  - 2.4.1
-  - 2.4.2
-  - jruby-head
-script: bundle exec rspec spec/
-env:
-  global:
-    - JRUBY_OPTS='--debug'
-install: bundle install --jobs=3 --retry=3
+cache: bundler
+
+git:
+  depth: false
+
+test: &test
+  stage: Test
+  language: ruby
+  before_install:
+    - gem install bundler
+    - gem update --system
+  script: bundle exec rspec
+
+jobs:
+  include:
+    - <<: *test
+      rvm: 2.6.3
+    - <<: *test
+      rvm: 2.5.5
+
+    - stage: coditsu
+      language: ruby
+      rvm: 2.6.3
+      before_install:
+        - gem update --system
+        - gem install bundler
+      script: \curl -sSL https://api.coditsu.io/run/ci | bash
+
+stages:
+  - coditsu
+  - test

data/CHANGELOG.md

@@ -1,6 +1,154 @@
 # Karafka framework changelog
 
-## 1.1.0 Unreleased
+## 1.3.0 (2019-09-09)
+- Drop support for Ruby 2.4
+- YARD docs tags cleanup
+
+## 1.3.0.rc1 (2019-07-31)
+- Drop support for Kafka 0.10 in favor of native support for Kafka 0.11.
+- Update ruby-kafka to the 0.7 version
+- Support messages headers receiving
+- Message bus unification
+- Parser available in metadata
+- Cleanup towards moving to a non-global state app management
+- Drop Ruby 2.3 support
+- Support for Ruby 2.6.3
+- `Karafka::Loader` has been removed in favor of Zeitwerk
+- Schemas are now contracts
+- #393 - Reorganize responders - removed `multiple_usage` constrain
+- #388 - ssl_client_cert_chain sync
+- #300 - Store value in a value key and replace its content with parsed version - without root merge
+- #331 - Disallow building groups without topics
+- #340 - Instrumentation unification. Better and more consistent naming
+- #340 - Procline instrumentation for a nicer process name
+- #342 - Change default for `fetcher_max_queue_size` from `100` to `10` to lower max memory usage
+- #345 - Cleanup exceptions names
+- #341 - Split connection delegator into batch delegator and single_delegator
+- #351 - Rename `#retrieve!` to `#parse!` on params and `#parsed` to `parse!` on params batch.
+- #351 - Adds '#first' for params_batch that returns parsed first element from the params_batch object.
+- #360 - Single params consuming mode automatically parses data specs
+- #359 - Divide mark_as_consumed into mark_as_consumed and mark_as_consumed!
+- #356 - Provide a `#values` for params_batch to extract only values of objects from the params_batch
+- #363 - Too shallow ruby-kafka version lock
+- #354 - Expose consumer heartbeat
+- #377 - Remove the persistent setup in favor of persistence
+- #375 - Sidekiq Backend parser mismatch
+- #369 - Single consumer can support more than one topic
+- #288 - Drop dependency on `activesupport` gem
+- #371 - SASL over SSL
+- #392 - Move params redundant data to metadata
+- #335 - Metadata access from within the consumer
+- #402 - Delayed reconnection upon critical failures
+- #405 - `reconnect_timeout` value is now being validated
+- #437 - Specs ensuring that the `#437` won't occur in the `1.3` release
+- #426 - ssl client cert key password
+- #444 - add certificate and private key validation
+- #460 - Decouple responder "parser" (generator?) from topic.parser (benissimo)
+- #463 - Split parsers into serializers / deserializers
+- #473 - Support SASL OAuthBearer Authentication
+- #475 - Disallow subscribing to the same topic with multiple consumers
+- #485 - Setting shutdown_timeout to nil kills the app without waiting for anything
+- #487 - Make listeners as instances
+- #29 - Consumer class names must have the word "Consumer" in it in order to work (Sidekiq backend)
+- #491 - irb is missing for console to work
+- #502 - Karafka process hangs when sending multiple sigkills
+- #506 - ssl_verify_hostname sync
+- #483 - Upgrade dry-validation before releasing 1.3
+- #492 - Use Zeitwerk for code reload in development
+- #508 - Reset the consumers instances upon reconnecting to a cluster
+- [#530](https://github.com/karafka/karafka/pull/530) - expose ruby and ruby-kafka version
+- [534](https://github.com/karafka/karafka/pull/534) - Allow to use headers in the deserializer object
+
+## 1.2.11
+- [#470](https://github.com/karafka/karafka/issues/470) Karafka not working with dry-configurable 0.8
+
+## 1.2.10
+- [#453](https://github.com/karafka/karafka/pull/453) require `Forwardable` module
+
+## 1.2.9
+- Critical exceptions now will cause consumer to stop instead of retrying without a break
+- #412 - Fix dry-inflector dependency lock in gemspec
+- #414 - Backport to 1.2 the delayed retry upon failure
+- #437 - Raw message is no longer added to params after ParserError raised
+
+## 1.2.8
+- #408 - Responder Topic Lookup Bug on Heroku
+
+## 1.2.7
+- Unlock Ruby-kafka version with a warning
+
+## 1.2.6
+- Lock WaterDrop to 1.2.3
+- Lock Ruby-Kafka to 0.6.x (support for 0.7 will be added in Karafka 1.3)
+- #382 - Full logging with AR, etc for development mode when there is Rails integration
+
+## 1.2.5
+- #354 - Expose consumer heartbeat
+- #373 - Async producer not working properly with responders
+
+## 1.2.4
+- #332 - Fetcher for max queue size
+
+## 1.2.3
+- #313 - support PLAINTEXT and SSL for scheme
+- #288 - drop activesupport callbacks in favor of notifications
+- #320 - Pausing indefinetely with nil pause timeout doesn't work
+- #318 - Partition pausing doesn't work with custom topic mappers
+- Rename ConfigAdapter to ApiAdapter to better reflect what it does
+- #317 - Manual offset committing doesn't work with custom topic mappers
+- #319 - Support for exponential backoff in pause
+
+## 1.2.2
+- #312 - Broken for ActiveSupport 5.2.0
+
+## 1.2.1
+- #304 - Unification of error instrumentation event details
+- #306 - Using file logger from within a trap context upon shutdown is impossible
+
+## 1.2.0
+- Spec improvements
+- #260 - Specs missing randomization
+- #251 - Shutdown upon non responding (unreachable) cluster is not possible
+- #258 - Investigate lowering requirements on activesupport
+- #246 - Alias consumer#mark_as_consumed on controller
+- #259 - Allow forcing key/partition key on responders
+- #267 - Styling inconsistency
+- #242 - Support setting the max bytes to fetch per request
+- #247 - Support SCRAM once released
+- #271 - Provide an after_init option to pass a configuration block
+- #262 - Error in the monitor code for NewRelic
+- #241 - Performance metrics
+- #274 - Rename controllers to consumers
+- #184 - Seek to
+- #284 - Dynamic Params parent class
+- #275 - ssl_ca_certs_from_system
+- #296 - Instrument forceful exit with an error
+- Replaced some of the activesupport parts with dry-inflector
+- Lower ActiveSupport dependency
+- Remove configurators in favor of the after_init block configurator
+- Ruby 2.5.0 support
+- Renamed Karafka::Connection::Processor to Karafka::Connection::Delegator to match incoming naming conventions
+- Renamed Karafka::Connection::Consumer to Karafka::Connection::Client due to #274
+- Removed HashWithIndifferentAccess in favor of a regular hash
+- JSON parsing defaults now to string keys
+- Lower memory usage due to less params data internal details
+- Support multiple ```after_init``` blocks in favor of a single one
+- Renamed ```received_at``` to ```receive_time``` to follow ruby-kafka and WaterDrop conventions
+- Adjust internal setup to easier map Ruby-Kafka config changes
+- System callbacks reorganization
+- Added ```before_fetch_loop``` configuration block for early client usage (```#seek```, etc)
+- Renamed ```after_fetched``` to ```after_fetch``` to normalize the naming convention
+- Instrumentation on a connection delegator level
+- Added ```params_batch#last``` method to retrieve last element after unparsing
+- All params keys are now strings
+
+## 1.1.2
+- #256 - Default kafka.seed_brokers configuration is created in invalid format
+
+## 1.1.1
+- #253 - Allow providing a global per app parser in config settings
+
+## 1.1.0
 - Gem bump
 - Switch from Celluloid to native Thread management
 - Improved shutdown process
@@ -14,7 +162,7 @@
 - Fixed accessibility of ```#params_batch``` from the outside of the controller
 - connection_pool config options are no longer required
 - celluloid config options are no longer required
-- ```#perform``` is not renamed to ```#consume``` with warning level on using the old one (deprecated)
+- ```#perform``` is now renamed to ```#consume``` with warning level on using the old one (deprecated)
 - #235 - Rename perform to consume
 - Upgrade to ruby-kafka 0.5
 - Due to redesign of Waterdrop concurrency setting is no longer needed
@@ -23,6 +171,7 @@
 - Renamed ```batch_consuming``` option to ```batch_fetching``` as it is not a consumption (with processing) but a process of fetching messages from Kafka. The messages is considered consumed, when it is processed.
 - Renamed ```batch_processing``` to ```batch_consuming``` to resemble Kafka concept of consuming messages.
 - Renamed ```after_received``` to ```after_fetched``` to normalize the naming conventions.
+- Responders support the per topic ```async``` option.
 
 ## 1.0.1
 - #210 - LoadError: cannot load such file -- [...]/karafka.rb

data/CONTRIBUTING.md

@@ -4,12 +4,11 @@
 
 First, thank you for considering contributing to karafka! It's people like you that make the open source community such a great community! 😊
 
-We welcome any type of contribution, not only code. You can help with 
+We welcome any type of contribution, not only code. You can help with:
 - **QA**: file bug reports, the more details you can give the better (e.g. screenshots with the console open)
 - **Marketing**: writing blog posts, howto's, printing stickers, ...
 - **Community**: presenting the project at meetups, organizing a dedicated meetup for the local community, ...
 - **Code**: take a look at the [open issues](issues). Even if you can't write code, commenting on them, showing that you care about a given issue matters. It helps us triage them.
-- **Money**: we welcome financial contributions in full transparency on our [open collective](https://opencollective.com/karafka).
 
 ## Your First Contribution
 
@@ -21,13 +20,13 @@ Any code change should be submitted as a pull request. The description should ex
 
 ## Code review process
 
-The bigger the pull request, the longer it will take to review and merge. Try to break down large pull requests in smaller chunks that are easier to review and merge.
-It is also always helpful to have some context for your pull request. What was the purpose? Why does it matter to you?
+Each pull request must pass all the rspec specs and meet our quality requirements.
 
-## Financial contributions
+To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
 
-We also welcome financial contributions in full transparency on our [open collective](https://opencollective.com/karafka).
-Anyone can file an expense. If the expense makes sense for the development of the community, it will be "merged" in the ledger of our open collective by the core contributors and the person who filed the expense will be reimbursed.
+Coditsu will automatically check your work against our quality standards. You can find your commit check results on the [builds page](https://app.coditsu.io/karafka/commit_builds) of Karafka organization.
+
+[![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/karafka/commit_builds)
 
 ## Questions
 

data/Gemfile

@@ -4,9 +4,9 @@ source 'https://rubygems.org'
 
 gemspec
 
-group :development, :test do
-  gem 'waterdrop'
-  gem 'timecop'
+group :test do
+  gem 'byebug'
+  gem 'factory_bot'
   gem 'rspec'
   gem 'simplecov'
 end

data/Gemfile.lock

@@ -1,111 +1,137 @@
 PATH
   remote: .
   specs:
-    karafka (1.1.0)
-      activesupport (>= 5.0)
-      dry-configurable (~> 0.7)
-      dry-validation (~> 0.11)
-      envlogic (~> 1.0)
+    karafka (1.3.0)
+      dry-configurable (~> 0.8)
+      dry-inflector (~> 0.1)
+      dry-monitor (~> 0.3)
+      dry-validation (~> 1.2)
+      envlogic (~> 1.1)
+      irb (~> 1.0)
       multi_json (>= 1.12)
       rake (>= 11.3)
-      require_all (>= 1.4)
-      ruby-kafka (>= 0.5)
-      thor (~> 0.19)
-      waterdrop (>= 1.0.1)
+      ruby-kafka (>= 0.7.8)
+      thor (~> 0.20)
+      waterdrop (~> 1.3.0)
+      zeitwerk (~> 2.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.4)
+    activesupport (6.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    concurrent-ruby (1.0.5)
-    delivery_boy (0.2.2)
-      king_konf (~> 0.1.8)
-      ruby-kafka (~> 0.4)
+      zeitwerk (~> 2.1, >= 2.1.8)
+    byebug (11.0.1)
+    concurrent-ruby (1.1.5)
+    delivery_boy (0.2.8)
+      king_konf (~> 0.3)
+      ruby-kafka (~> 0.7.8)
     diff-lcs (1.3)
-    docile (1.1.5)
-    dry-configurable (0.7.0)
+    digest-crc (0.4.1)
+    docile (1.3.2)
+    dry-configurable (0.8.3)
       concurrent-ruby (~> 1.0)
-    dry-container (0.6.0)
+      dry-core (~> 0.4, >= 0.4.7)
+    dry-container (0.7.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.1)
+    dry-core (0.4.9)
+      concurrent-ruby (~> 1.0)
+    dry-equalizer (0.2.2)
+    dry-events (0.2.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 0.4)
+      dry-equalizer (~> 0.2)
+    dry-inflector (0.1.2)
+    dry-initializer (3.0.1)
+    dry-logic (1.0.3)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.2.0)
-    dry-logic (0.4.2)
-      dry-container (~> 0.2, >= 0.2.6)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
-    dry-types (0.12.2)
+    dry-monitor (0.3.1)
+      dry-configurable (~> 0.5)
+      dry-core (~> 0.4)
+      dry-equalizer (~> 0.2)
+      dry-events (~> 0.1)
+    dry-schema (1.3.3)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1)
-      dry-container (~> 0.3)
-      dry-core (~> 0.2, >= 0.2.1)
+      dry-configurable (~> 0.8, >= 0.8.3)
+      dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-      dry-logic (~> 0.4, >= 0.4.2)
-      inflecto (~> 0.0.0, >= 0.0.2)
-    dry-validation (0.11.1)
+      dry-initializer (~> 3.0)
+      dry-logic (~> 1.0)
+      dry-types (~> 1.0)
+    dry-types (1.1.1)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1, >= 0.1.3)
-      dry-core (~> 0.2, >= 0.2.1)
+      dry-container (~> 0.3)
+      dry-core (~> 0.4, >= 0.4.4)
+      dry-equalizer (~> 0.2, >= 0.2.2)
+      dry-inflector (~> 0.1, >= 0.1.2)
+      dry-logic (~> 1.0, >= 1.0.2)
+    dry-validation (1.3.1)
+      concurrent-ruby (~> 1.0)
+      dry-container (~> 0.7, >= 0.7.1)
+      dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-      dry-logic (~> 0.4, >= 0.4.0)
-      dry-types (~> 0.12.0)
-    envlogic (1.0.4)
-      activesupport
-    i18n (0.9.1)
+      dry-initializer (~> 3.0)
+      dry-schema (~> 1.0, >= 1.3.1)
+    envlogic (1.1.0)
+      dry-inflector (~> 0.1)
+    factory_bot (5.0.2)
+      activesupport (>= 4.2.0)
+    i18n (1.6.0)
       concurrent-ruby (~> 1.0)
-    inflecto (0.0.2)
-    json (2.1.0)
-    king_konf (0.1.8)
-    minitest (5.10.3)
-    multi_json (1.12.2)
-    null-logger (0.1.4)
-    rake (12.2.1)
-    require_all (1.4.0)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.0)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    irb (1.0.0)
+    json (2.2.0)
+    king_konf (0.3.7)
+    minitest (5.11.3)
+    multi_json (1.13.1)
+    rake (12.3.3)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.2)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.0)
-    ruby-kafka (0.5.0)
-    simplecov (0.15.1)
-      docile (~> 1.1.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.2)
+    ruby-kafka (0.7.10)
+      digest-crc
+    simplecov (0.17.0)
+      docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    thor (0.20.0)
+    thor (0.20.3)
     thread_safe (0.3.6)
-    timecop (0.9.1)
-    tzinfo (1.2.4)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    waterdrop (1.0.1)
-      delivery_boy (>= 0.2.2)
-      dry-configurable (~> 0.7)
-      dry-validation (~> 0.11)
-      null-logger
-      ruby-kafka (>= 0.5)
+    waterdrop (1.3.0)
+      delivery_boy (~> 0.2)
+      dry-configurable (~> 0.8)
+      dry-monitor (~> 0.3)
+      dry-validation (~> 1.2)
+      ruby-kafka (>= 0.7.8)
+      zeitwerk (~> 2.1)
+    zeitwerk (2.1.10)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  byebug
+  factory_bot
   karafka!
   rspec
   simplecov
-  timecop
-  waterdrop
 
 BUNDLED WITH
-   1.15.4
+   2.0.2