karafka 0.5.0.3 → 0.6.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6d336fa4ee7c6d7a9b38ee1ea08e70a9f6208312
-  data.tar.gz: 219b4f431dc21cbe403f2f8aa50b81eae850c93c
+  metadata.gz: 2b23b384c41e678138f2dec19e303e3cd62dc537
+  data.tar.gz: 794671bee86cd69916ae16f48111179be4e45228
 SHA512:
-  metadata.gz: 288e9e7054f1716c9d2600fb572b4fd288c6d33a29df89b644d584fb91d410da40ace8628889d7d936bd0526b40f0ffcf75729a41826c1bd0b6ddd1e5262e5d1
-  data.tar.gz: 00214b9cf81f378b42d9a3c78151f1a8f95cfe5a8110c7566ae2280edf41bbcd9cc527e586d7d3978195418c1b8d6e48821a7ac21855c1578ca193e78c484241
+  metadata.gz: 29f049589ec7f572b9de747c005522a2b40076b5fc8425f451f83f12d8eef549b6ce403cf2201068ca462dedae83df799735f4655df167714116a6990238a7df
+  data.tar.gz: 436c792d7b15fafd68754c75bc558559ed2d0828591aba5a0644a44a720dbf4bffeefa428d146517552ba6d66dfa266d76ab9b3a3fa1c6e3a19c685a4496eaf6

data/.console_irbrc

@@ -0,0 +1,13 @@
+# irbrc for Karafka console
+require 'karafka'
+require Karafka.boot_file
+
+IRB.conf[:AUTO_INDENT] = true
+IRB.conf[:SAVE_HISTORY] = 1000
+IRB.conf[:USE_READLINE] = true
+IRB.conf[:HISTORY_FILE] = "#{Karafka::App.root}/.irb-history"
+IRB.conf[:LOAD_MODULES] = [] unless IRB.conf.key?(:LOAD_MODULES)
+
+unless IRB.conf[:LOAD_MODULES].include?('irb/completion')
+  IRB.conf[:LOAD_MODULES] << 'irb/completion'
+end

data/.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,2 @@
+<!-- Love karafka? Please consider supporting our collective:
+👉  https://opencollective.com/karafka/donate -->

data/.gitignore

@@ -18,6 +18,7 @@ db/*.sqlite3
 **.war
 *.rbc
 *.sassc
+.byebug_history
 .redcar/
 .capistrano/
 .sass-cache

data/CHANGELOG.md

@@ -1,5 +1,64 @@
 # Karafka framework changelog
 
+## 0.6.0.rc1
+
+### Closed issues:
+
+- #103 - Env for logger is loaded 2 early (on gem load not on app init)
+- #142 - Possibility to better control Kafka consumers (consumer groups management)
+- #150 - Add support for start_from_beginning on a per topic basis
+- #154 - Support for min_bytes and max_wait_time on messages consuming
+- #160 - Reorganize settings to better resemble ruby-kafka requirements
+- #164 - If we decide to have configuration per topic, topic uniqueness should be removed
+- #165 - Router validator
+- #166 - Params and route reorganization (new API)
+- #167 - Remove Sidekiq UI from Karafka
+- #168 - Introduce unique IDs of routes
+- #171 - Add kafka message metadata to params
+- #176 - Transform Karafka::Connection::Consumer into a module
+- #177 - Monitor not reacting when kafka killed with -9
+- #175 - Allow single consumer to subscribe to multiple topics
+- #178 - Remove parsing failover when cannot unparse data
+- #174 - Extended config validation
+- #180 - Switch from JSON parser to yajl-ruby
+- #181 - When responder is defined and not used due to ```respond_with``` not being triggered in the perform, it won't raise an exception.
+- #188 - Rename name in config to client id
+- #186 - Support ruby-kafka ```ssl_ca_cert_file_path``` config
+- #189 - karafka console does not preserve history on exit
+
+### New features and improvements
+
+- batch processing thanks to ```#batch_processing``` flag and ```#params_batch``` on controllers
+- ```#topic``` method on an controller instance to make a clear distinction in between params and route details
+- Changed routing model (still compatible with 0.5) to allow better resources management
+- Lower memory requirements due to object creation limitation (2-3 times less objects on each new message)
+- Introduced the ```#batch_processing``` config flag (config for #126) that can be set per each consumer_group
+- Added support for partition, offset and partition key in the params hash
+- ```name``` option in config renamed to ```client_id```
+
+### Incompatibilities
+
+- Removed worker glass as dependency (now and independent gem)
+- ```kafka.hosts``` option renamed to ```kafka.seed_brokers``` - you don't need to provide all the hosts to work with Kafka
+- ```start_from_beginning``` moved into kafka scope (```kafka.start_from_beginning```)
+- Router no longer checks for route uniqueness - now you can define same routes for multiple kafkas and do a lot of crazy stuff, so it's your responsibility to check uniqueness
+- Change in the way we identify topics in between Karafka and Sidekiq workers. If you upgrade, please make sure, all the jobs scheduled in Sidekiq are finished before the upgrade.
+- ```batch_mode``` renamed to ```batch_consuming```
+- Renamed content to value to better resemble ruby-kafka internal messages naming convention
+- When having a responder with ```required``` topics and not using ```#respond_with``` at all, it will raise an exception
+- Renamed ```inline_mode``` to ```inline_processing``` to resemble other settings conventions
+
+### Other changes
+- PolishGeeksDevTools removed (in favour of Coditsu)
+- Waaaaaay better code quality thanks to switching from dev tools to Coditsu
+- Gem bump
+- Cleaner internal API
+- SRP
+- Better settings proxying and management between ruby-kafka and karafka
+- All internal validations are now powered by dry-validation
+- Better naming conventions to reflect Kafka reality
+- Removed Karafka::Connection::Message in favour of direct message details extraction from Kafka::FetchedMessage
+
 ## 0.5.0.3
 - #132 - When Kafka is gone, should reconnect after a time period
 - #136 - new ruby-kafka version + other gem bumps
@@ -247,4 +306,3 @@
 
 ## 0.1.0
 - Initial framework code
-

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,46 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maciej@coditsu.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/CONTRIBUTING.md

@@ -0,0 +1,67 @@
+# Contribute
+
+## Introduction
+
+First, thank you for considering contributing to karafka! It's people like you that make the open source community such a great community! 😊
+
+We welcome any type of contribution, not only code. You can help with 
+- **QA**: file bug reports, the more details you can give the better (e.g. screenshots with the console open)
+- **Marketing**: writing blog posts, howto's, printing stickers, ...
+- **Community**: presenting the project at meetups, organizing a dedicated meetup for the local community, ...
+- **Code**: take a look at the [open issues](issues). Even if you can't write code, commenting on them, showing that you care about a given issue matters. It helps us triage them.
+- **Money**: we welcome financial contributions in full transparency on our [open collective](https://opencollective.com/karafka).
+
+## Your First Contribution
+
+Working on your first Pull Request? You can learn how from this *free* series, [How to Contribute to an Open Source Project on GitHub](https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github).
+
+## Submitting code
+
+Any code change should be submitted as a pull request. The description should explain what the code does and give steps to execute it. The pull request should also contain tests.
+
+## Code review process
+
+The bigger the pull request, the longer it will take to review and merge. Try to break down large pull requests in smaller chunks that are easier to review and merge.
+It is also always helpful to have some context for your pull request. What was the purpose? Why does it matter to you?
+
+## Financial contributions
+
+We also welcome financial contributions in full transparency on our [open collective](https://opencollective.com/karafka).
+Anyone can file an expense. If the expense makes sense for the development of the community, it will be "merged" in the ledger of our open collective by the core contributors and the person who filed the expense will be reimbursed.
+
+## Questions
+
+If you have any questions, create an [issue](issue) (protip: do a quick search first to see if someone else didn't ask the same question before!).
+You can also reach us at hello@karafka.opencollective.com.
+
+## Credits
+
+### Contributors
+
+Thank you to all the people who have already contributed to karafka!
+<a href="graphs/contributors"><img src="https://opencollective.com/karafka/contributors.svg?width=890" /></a>
+
+
+### Backers
+
+Thank you to all our backers! [[Become a backer](https://opencollective.com/karafka#backer)]
+
+<a href="https://opencollective.com/karafka#backers" target="_blank"><img src="https://opencollective.com/karafka/backers.svg?width=890"></a>
+
+
+### Sponsors
+
+Thank you to all our sponsors! (please ask your company to also support this open source project by [becoming a sponsor](https://opencollective.com/karafka#sponsor))
+
+<a href="https://opencollective.com/karafka/sponsor/0/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/0/avatar.svg"></a>
+<a href="https://opencollective.com/karafka/sponsor/1/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/1/avatar.svg"></a>
+<a href="https://opencollective.com/karafka/sponsor/2/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/2/avatar.svg"></a>
+<a href="https://opencollective.com/karafka/sponsor/3/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/3/avatar.svg"></a>
+<a href="https://opencollective.com/karafka/sponsor/4/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/4/avatar.svg"></a>
+<a href="https://opencollective.com/karafka/sponsor/5/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/5/avatar.svg"></a>
+<a href="https://opencollective.com/karafka/sponsor/6/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/6/avatar.svg"></a>
+<a href="https://opencollective.com/karafka/sponsor/7/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/7/avatar.svg"></a>
+<a href="https://opencollective.com/karafka/sponsor/8/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/8/avatar.svg"></a>
+<a href="https://opencollective.com/karafka/sponsor/9/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/9/avatar.svg"></a>
+
+<!-- This `CONTRIBUTING.md` is based on @nayafia's template https://github.com/nayafia/contributing-template -->

data/Gemfile

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gemspec
 
 group :development, :test do
-  gem 'polishgeeks-dev-tools'
   gem 'timecop'
 end

data/Gemfile.lock

@@ -1,37 +1,28 @@
 PATH
   remote: .
   specs:
-    karafka (0.5.0.3)
-      activesupport (~> 5.0)
-      celluloid (~> 0.17)
+    karafka (0.6.0.rc1)
+      activesupport (>= 5.0)
+      celluloid
       dry-configurable (~> 0.7)
-      dry-validation (~> 0.10.6)
+      dry-validation (~> 0.11)
       envlogic (~> 1.0)
-      rake (~> 11.3)
-      ruby-kafka (= 0.3.17)
+      rake (>= 11.3)
+      ruby-kafka (>= 0.4)
       sidekiq (>= 4.2)
       thor (~> 0.19)
-      waterdrop (~> 0.3.2.4)
-      worker-glass (~> 0.2)
+      waterdrop (>= 0.4)
+      yajl-ruby (>= 1.3.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.0.2)
+    activesupport (5.1.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    addressable (2.4.0)
-    ast (2.3.0)
-    axiom-types (0.1.1)
-      descendants_tracker (~> 0.0.4)
-      ice_nine (~> 0.11.0)
-      thread_safe (~> 0.3, >= 0.3.1)
-    brakeman (3.3.3)
-    bundler-audit (0.5.0)
-      bundler (~> 1.2)
-      thor (~> 0.18)
+    byebug (9.0.6)
     celluloid (0.17.3)
       celluloid-essentials
       celluloid-extras
@@ -49,17 +40,9 @@ GEM
       timers (>= 4.1.1)
     celluloid-supervision (0.20.6)
       timers (>= 4.1.1)
-    codeclimate-engine-rb (0.3.1)
-      virtus (~> 1.0)
-    coderay (1.1.1)
-    coercible (1.0.0)
-      descendants_tracker (~> 0.0.1)
-    colorize (0.8.1)
     concurrent-ruby (1.0.5)
     connection_pool (2.2.1)
-    descendants_tracker (0.0.4)
-      thread_safe (~> 0.3, >= 0.3.1)
-    diff-lcs (1.2.5)
+    diff-lcs (1.3)
     docile (1.1.5)
     dry-configurable (0.7.0)
       concurrent-ruby (~> 1.0)
@@ -73,7 +56,7 @@ GEM
       dry-container (~> 0.2, >= 0.2.6)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
-    dry-types (0.10.3)
+    dry-types (0.11.1)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1)
       dry-container (~> 0.3)
@@ -81,163 +64,79 @@ GEM
       dry-equalizer (~> 0.2)
       dry-logic (~> 0.4, >= 0.4.0)
       inflecto (~> 0.0.0, >= 0.0.2)
-    dry-validation (0.10.7)
+    dry-validation (0.11.0)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
       dry-core (~> 0.2, >= 0.2.1)
       dry-equalizer (~> 0.2)
       dry-logic (~> 0.4, >= 0.4.0)
-      dry-types (~> 0.9, >= 0.9.0)
+      dry-types (~> 0.11.0)
     envlogic (1.0.3)
       activesupport
-    equalizer (0.0.11)
-    erubis (2.7.0)
-    faker (1.6.6)
-      i18n (~> 0.5)
-    flay (2.8.0)
-      erubis (~> 2.7.0)
-      path_expander (~> 1.0)
-      ruby_parser (~> 3.0)
-      sexp_processor (~> 4.0)
-    flog (4.4.0)
-      path_expander (~> 1.0)
-      ruby_parser (~> 3.1, > 3.1.0)
-      sexp_processor (~> 4.4)
-    haml (4.0.7)
-      tilt
-    haml_lint (0.18.1)
-      haml (~> 4.0)
-      rake (>= 10, < 12)
-      rubocop (>= 0.36.0)
-      sysexits (~> 1.1)
-    hitimes (1.2.5)
-    i18n (0.7.0)
-    ice_nine (0.11.2)
+    ffi (1.9.18)
+    gssapi (1.2.0)
+      ffi (>= 1.0.1)
+    hitimes (1.2.6)
+    i18n (0.8.6)
     inflecto (0.0.2)
-    json (2.0.2)
-    launchy (2.4.3)
-      addressable (~> 2.3)
-    method_source (0.8.2)
-    minitest (5.10.1)
+    json (2.1.0)
+    minitest (5.10.3)
     null-logger (0.1.4)
-    parser (2.3.1.2)
-      ast (~> 2.2)
-    path_expander (1.0.0)
-    polishgeeks-dev-tools (1.3.2)
-      brakeman
-      bundler-audit
-      faker
-      haml_lint
-      pry
-      rspec
-      rubocop
-      rubocop-rspec
-      rubycritic
-      shoulda
-      simplecov
-      timecop
-      yard
-    powerpack (0.1.1)
-    pry (0.10.4)
-      coderay (~> 1.1.0)
-      method_source (~> 0.8.1)
-      slop (~> 3.4)
     rack (2.0.3)
     rack-protection (2.0.0)
       rack
-    rainbow (2.1.0)
-    rake (11.3.0)
+    rake (12.0.0)
     redis (3.3.3)
-    reek (4.1.0)
-      codeclimate-engine-rb (~> 0.3.1)
-      parser (~> 2.3.1, >= 2.3.1.2)
-      rainbow (~> 2.0)
-    rspec (3.5.0)
-      rspec-core (~> 3.5.0)
-      rspec-expectations (~> 3.5.0)
-      rspec-mocks (~> 3.5.0)
-    rspec-core (3.5.2)
-      rspec-support (~> 3.5.0)
-    rspec-expectations (3.5.0)
+    rspec (3.6.0)
+      rspec-core (~> 3.6.0)
+      rspec-expectations (~> 3.6.0)
+      rspec-mocks (~> 3.6.0)
+    rspec-core (3.6.0)
+      rspec-support (~> 3.6.0)
+    rspec-expectations (3.6.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-mocks (3.5.0)
+      rspec-support (~> 3.6.0)
+    rspec-mocks (3.6.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-support (3.5.0)
-    rubocop (0.42.0)
-      parser (>= 2.3.1.1, < 3.0)
-      powerpack (~> 0.1)
-      rainbow (>= 1.99.1, < 3.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
-    rubocop-rspec (1.5.3)
-      rubocop (>= 0.42.0)
-    ruby-kafka (0.3.17)
-    ruby-progressbar (1.8.1)
-    ruby_parser (3.8.2)
-      sexp_processor (~> 4.1)
-    rubycritic (2.9.2)
-      colorize
-      flay (= 2.8.0)
-      flog (= 4.4.0)
-      launchy (= 2.4.3)
-      parser (= 2.3.1.2)
-      reek (= 4.1.0)
-      ruby_parser (~> 3.8)
-      virtus (~> 1.0)
-    sexp_processor (4.7.0)
-    shoulda (3.5.0)
-      shoulda-context (~> 1.0, >= 1.0.1)
-      shoulda-matchers (>= 1.4.1, < 3.0)
-    shoulda-context (1.2.1)
-    shoulda-matchers (2.8.0)
-      activesupport (>= 3.0.0)
-    sidekiq (5.0.3)
+      rspec-support (~> 3.6.0)
+    rspec-support (3.6.0)
+    ruby-kafka (0.4.0)
+      gssapi (>= 1.2.0)
+    sidekiq (5.0.4)
       concurrent-ruby (~> 1.0)
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
       redis (~> 3.3, >= 3.3.3)
-    simplecov (0.12.0)
+    simplecov (0.15.0)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.0)
-    slop (3.6.0)
-    sysexits (1.2.0)
+    simplecov-html (0.10.2)
     thor (0.19.4)
-    thread_safe (0.3.5)
-    tilt (2.0.5)
-    timecop (0.8.1)
+    thread_safe (0.3.6)
+    timecop (0.9.1)
     timers (4.1.2)
       hitimes
     tzinfo (1.2.3)
       thread_safe (~> 0.1)
-    unicode-display_width (1.1.0)
-    virtus (1.0.5)
-      axiom-types (~> 0.1)
-      coercible (~> 1.0)
-      descendants_tracker (~> 0.0, >= 0.0.3)
-      equalizer (~> 0.0, >= 0.0.9)
-    waterdrop (0.3.2.4)
+    waterdrop (0.4.0)
       bundler
       connection_pool
       dry-configurable (~> 0.6)
       null-logger
       rake
-      ruby-kafka
-    worker-glass (0.2.3)
-      activesupport
-      null-logger
-    yard (0.9.5)
+      ruby-kafka (~> 0.4)
+    yajl-ruby (1.3.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 1.2)
+  byebug
   karafka!
-  polishgeeks-dev-tools
+  rspec (>= 3.6)
+  simplecov (>= 0.14)
   timecop
 
 BUNDLED WITH