karafka 0.6.0.rc1 → 0.6.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2b23b384c41e678138f2dec19e303e3cd62dc537
-  data.tar.gz: 794671bee86cd69916ae16f48111179be4e45228
+  metadata.gz: f1ac31c4f943edba6e25886bc34b2bfad35e69fb
+  data.tar.gz: 488973afae1eb933b27dec7dd2fdcbb21fe6edae
 SHA512:
-  metadata.gz: 29f049589ec7f572b9de747c005522a2b40076b5fc8425f451f83f12d8eef549b6ce403cf2201068ca462dedae83df799735f4655df167714116a6990238a7df
-  data.tar.gz: 436c792d7b15fafd68754c75bc558559ed2d0828591aba5a0644a44a720dbf4bffeefa428d146517552ba6d66dfa266d76ab9b3a3fa1c6e3a19c685a4496eaf6
+  metadata.gz: 7e0ce78c37d1309d7b6adc51b4c59545df6eed3ee1dca0441c751b309090bd4d62e9da100fbe8fb7086af1b4f1d600ead0f9f7b873bf3d45e8f95092c30cdd76
+  data.tar.gz: d2555a9f9ca030df2d31a49d2a272e4d13b681ad53590708c6ed960004690761a0e3cf9e88ee000a0870d48d60e8987a7c0bf619c10cbfc5dad057fb374115b3

data/.travis.yml

@@ -1,10 +1,16 @@
 language: ruby
+sudo: false
 rvm:
   - 2.3.0
   - 2.3.1
   - 2.3.2
   - 2.3.3
+  - 2.3.4
   - 2.4.0
   - 2.4.1
-script:
-  - bundle exec rake
+  - jruby-9.1.12.0
+script: bundle exec rspec spec/
+env:
+  global:
+    - JRUBY_OPTS='--debug'
+install: bundle install --jobs=3 --retry=3

data/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Karafka framework changelog
 
-## 0.6.0.rc1
+## 0.6.0.rc2 - released
 
 ### Closed issues:
 
@@ -20,11 +20,20 @@
 - #175 - Allow single consumer to subscribe to multiple topics
 - #178 - Remove parsing failover when cannot unparse data
 - #174 - Extended config validation
-- #180 - Switch from JSON parser to yajl-ruby
+- ~~#180 - Switch from JSON parser to yajl-ruby~~
 - #181 - When responder is defined and not used due to ```respond_with``` not being triggered in the perform, it won't raise an exception.
 - #188 - Rename name in config to client id
 - #186 - Support ruby-kafka ```ssl_ca_cert_file_path``` config
 - #189 - karafka console does not preserve history on exit
+- #191 - Karafka 0.6.0rc1 does not work with jruby / now it does :-)
+- Switch to multi json so everyone can use their favourite JSON parser
+- Added jruby support in general and in Travis
+- #196 - Topic mapper does not map topics when subscribing thanks to @webandtech
+- #96 - Karafka server - possiblity to run it only for a certain topics
+- ~~karafka worker cli option is removed (please use sidekiq directly)~~ - restored, bad idea
+- (optional) pausing upon processing failures ```pause_timeout```
+- Karafka console main process no longer intercepts irb errors
+- Wiki updates
 
 ### New features and improvements
 
@@ -38,6 +47,7 @@
 
 ### Incompatibilities
 
+- Default boot file is renamed from app.rb to karafka.rb
 - Removed worker glass as dependency (now and independent gem)
 - ```kafka.hosts``` option renamed to ```kafka.seed_brokers``` - you don't need to provide all the hosts to work with Kafka
 - ```start_from_beginning``` moved into kafka scope (```kafka.start_from_beginning```)
@@ -47,6 +57,7 @@
 - Renamed content to value to better resemble ruby-kafka internal messages naming convention
 - When having a responder with ```required``` topics and not using ```#respond_with``` at all, it will raise an exception
 - Renamed ```inline_mode``` to ```inline_processing``` to resemble other settings conventions
+- Renamed ```inline_processing``` to ```processing_backend``` to reach 1.0 future compatibility
 
 ### Other changes
 - PolishGeeksDevTools removed (in favour of Coditsu)

data/CONTRIBUTING.md

@@ -40,28 +40,3 @@ You can also reach us at hello@karafka.opencollective.com.
 
 Thank you to all the people who have already contributed to karafka!
 <a href="graphs/contributors"><img src="https://opencollective.com/karafka/contributors.svg?width=890" /></a>
-
-
-### Backers
-
-Thank you to all our backers! [[Become a backer](https://opencollective.com/karafka#backer)]
-
-<a href="https://opencollective.com/karafka#backers" target="_blank"><img src="https://opencollective.com/karafka/backers.svg?width=890"></a>
-
-
-### Sponsors
-
-Thank you to all our sponsors! (please ask your company to also support this open source project by [becoming a sponsor](https://opencollective.com/karafka#sponsor))
-
-<a href="https://opencollective.com/karafka/sponsor/0/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/0/avatar.svg"></a>
-<a href="https://opencollective.com/karafka/sponsor/1/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/1/avatar.svg"></a>
-<a href="https://opencollective.com/karafka/sponsor/2/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/2/avatar.svg"></a>
-<a href="https://opencollective.com/karafka/sponsor/3/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/3/avatar.svg"></a>
-<a href="https://opencollective.com/karafka/sponsor/4/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/4/avatar.svg"></a>
-<a href="https://opencollective.com/karafka/sponsor/5/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/5/avatar.svg"></a>
-<a href="https://opencollective.com/karafka/sponsor/6/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/6/avatar.svg"></a>
-<a href="https://opencollective.com/karafka/sponsor/7/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/7/avatar.svg"></a>
-<a href="https://opencollective.com/karafka/sponsor/8/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/8/avatar.svg"></a>
-<a href="https://opencollective.com/karafka/sponsor/9/website" target="_blank"><img src="https://opencollective.com/karafka/sponsor/9/avatar.svg"></a>
-
-<!-- This `CONTRIBUTING.md` is based on @nayafia's template https://github.com/nayafia/contributing-template -->

data/Gemfile

@@ -6,4 +6,6 @@ gemspec
 
 group :development, :test do
   gem 'timecop'
+  gem 'rspec'
+  gem 'simplecov'
 end

data/Gemfile.lock

@@ -1,18 +1,19 @@
 PATH
   remote: .
   specs:
-    karafka (0.6.0.rc1)
+    karafka (0.6.0.rc2)
       activesupport (>= 5.0)
       celluloid
       dry-configurable (~> 0.7)
       dry-validation (~> 0.11)
       envlogic (~> 1.0)
+      multi_json (>= 1.12)
       rake (>= 11.3)
+      require_all (>= 1.4)
       ruby-kafka (>= 0.4)
       sidekiq (>= 4.2)
       thor (~> 0.19)
       waterdrop (>= 0.4)
-      yajl-ruby (>= 1.3.0)
 
 GEM
   remote: https://rubygems.org/
@@ -22,7 +23,6 @@ GEM
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    byebug (9.0.6)
     celluloid (0.17.3)
       celluloid-essentials
       celluloid-extras
@@ -81,12 +81,14 @@ GEM
     inflecto (0.0.2)
     json (2.1.0)
     minitest (5.10.3)
+    multi_json (1.12.1)
     null-logger (0.1.4)
     rack (2.0.3)
     rack-protection (2.0.0)
       rack
     rake (12.0.0)
     redis (3.3.3)
+    require_all (1.4.0)
     rspec (3.6.0)
       rspec-core (~> 3.6.0)
       rspec-expectations (~> 3.6.0)
@@ -112,7 +114,7 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    thor (0.19.4)
+    thor (0.20.0)
     thread_safe (0.3.6)
     timecop (0.9.1)
     timers (4.1.2)
@@ -126,17 +128,14 @@ GEM
       null-logger
       rake
       ruby-kafka (~> 0.4)
-    yajl-ruby (1.3.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.2)
-  byebug
   karafka!
-  rspec (>= 3.6)
-  simplecov (>= 0.14)
+  rspec
+  simplecov
   timecop
 
 BUNDLED WITH

data/README.md

@@ -1,4 +1,4 @@
-![karafka logo](http://mensfeld.github.io/karafka-framework-introduction/img/karafka-04.png)
+![karafka logo](https://raw.githubusercontent.com/karafka/misc/master/logo/karafka_logotype_transparent2.png)
 
 [![Build Status](https://travis-ci.org/karafka/karafka.png)](https://travis-ci.org/karafka/karafka)
 [![Backers on Open Collective](https://opencollective.com/karafka/backers/badge.svg)](#backers) [![Sponsors on Open Collective](https://opencollective.com/karafka/sponsors/badge.svg)](#sponsors) [![Join the chat at https://gitter.im/karafka/karafka](https://badges.gitter.im/karafka/karafka.svg)](https://gitter.im/karafka/karafka?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
@@ -19,9 +19,28 @@ Karafka based applications can be easily deployed to any type of infrastructure,
 * Capistrano
 * Docker
 
-## Support
+## Getting started
+
+If you want to get started with Kafka and Karafka as fast as possible, then the best idea is to just clone our example repository:
+
+```bash
+git clone https://github.com/karafka/karafka-example-app ./example_app
+```
+
+then, just bundle install all the dependencies:
+
+```bash
+cd ./example_app
+bundle install
+```
+
+and follow the instructions from the [example app Wiki](https://github.com/karafka/karafka-example-app/blob/master/README.md).
+
+**Note**: you need to ensure, that you have Kafka up and running and you need to configure Kafka seed_brokers in the ```karafka.rb``` file.
+
+If you need more details and know how on how to start Karafka with a clean installation, read the [Getting started page](https://github.com/karafka/karafka/wiki/Getting-started) section of our Wiki.
 
-**Warning**: We're currently in the middle of upgrading our [Wiki pages](https://github.com/karafka/karafka/wiki), to match our newest 0.6 release and it's API. If you use the 0.5 version, you might encounter some incompatibilities. We're really sorry for the inconvenience.
+## Support
 
 Karafka has a [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything. It covers the whole installation, setup and deployment along with other useful details on how to run Karafka.
 
@@ -39,14 +58,6 @@ If you are interested in our commercial services, please contact [Maciej Mensfel
 
 Karafka framework and Karafka team are __not__ related to Kafka streaming service called CloudKarafka in any matter. We don't recommend nor discourage usage of their platform.
 
-## Requirements
-
-In order to use Karafka framework, you need to have:
-
-  - Zookeeper (required by Kafka)
-  - Kafka (at least 0.9.0)
-  - Ruby (at least 2.3.0)
-
 ## Note on Patches/Pull Requests
 
 Fork the project.

data/Rakefile

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
-require 'bundler'
-require 'rake'
+require 'rspec'
 require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)

data/bin/karafka

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'karafka'
-require Karafka.boot_file.to_s if File.exist?(Karafka.boot_file.to_s)
+require Karafka.boot_file.to_s
 
 Karafka::Cli.prepare
 Karafka::Cli.start

data/karafka.gemspec

@@ -26,12 +26,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activesupport', '>= 5.0'
   spec.add_dependency 'dry-validation', '~> 0.11'
   spec.add_dependency 'dry-configurable', '~> 0.7'
-  spec.add_dependency 'yajl-ruby', '>= 1.3.0'
-
-  spec.add_development_dependency 'bundler', '~> 1.2'
-  spec.add_development_dependency 'rspec', '>= 3.6'
-  spec.add_development_dependency 'simplecov', '>= 0.14'
-  spec.add_development_dependency 'byebug'
+  spec.add_dependency 'multi_json', '>= 1.12'
+  spec.add_dependency 'require_all', '>= 1.4'
 
   spec.required_ruby_version = '>= 2.3.0'
 

data/lib/karafka.rb

@@ -16,7 +16,8 @@
   envlogic
   thor
   fileutils
-  yajl
+  multi_json
+  require_all
   dry-configurable
   dry-validation
   active_support/callbacks
@@ -61,18 +62,18 @@ module Karafka
     end
 
     # @return [String] path to a default file that contains booting procedure etc
-    # @note By default it is a file called 'app.rb' but it can be specified as you wish if you
-    #   have Karafka that is merged into a Sinatra/Rails app and app.rb is taken.
+    # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
+    #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
     #   It will be used for console/workers/etc
     # @example Standard only-Karafka case
-    #   Karafka.boot_file #=> '/home/app_path/app.rb'
-    # @example Non standard case
-    #   KARAFKA_BOOT_FILE='/home/app_path/karafka.rb'
     #   Karafka.boot_file #=> '/home/app_path/karafka.rb'
+    # @example Non standard case
+    #   KARAFKA_BOOT_FILE='/home/app_path/app.rb'
+    #   Karafka.boot_file #=> '/home/app_path/app.rb'
     def boot_file
-      Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'app.rb'))
+      Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
     end
   end
 end
 
-Karafka::Loader.new.load!(Karafka.core_root)
+Karafka::Loader.load!(Karafka.core_root)

data/lib/karafka/attributes_map.rb

@@ -22,6 +22,7 @@ module Karafka
           ],
           subscription: %i[start_from_beginning max_bytes_per_partition],
           consuming: %i[min_bytes max_wait_time],
+          pausing: %i[pause_timeout],
           # All the options that are under kafka config namespace, but are not used
           # directly with kafka api, but from the Karafka user perspective, they are
           # still related to kafka. They should not be proxied anywhere
@@ -32,7 +33,7 @@ module Karafka
       # @return [Array<Symbol>] properties that can be set on a per topic level
       def topic
         (config_adapter[:subscription] + %i[
-          inline_processing
+          processing_backend
           name
           worker
           parser
@@ -52,7 +53,7 @@ module Karafka
         #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
         ignored_settings = config_adapter[:subscription]
         defined_settings = config_adapter.values.flatten
-        karafka_settings = %i[batch_consuming topic_mapper]
+        karafka_settings = %i[batch_consuming]
         # This is a drity and bad hack of dry-configurable to get keys before setting values
         dynamically_proxied = Karafka::Setup::Config
                               ._settings

data/lib/karafka/base_controller.rb

@@ -107,7 +107,14 @@ module Karafka
     # will schedule a call task in sidekiq
     def schedule
       run_callbacks :schedule do
-        topic.inline_processing ? call_inline : call_async
+        case topic.processing_backend
+        when :inline then
+          call_inline
+        when :sidekiq then
+          call_async
+        else
+          raise Errors::InvalidProcessingBackend, topic.processing_backend
+        end
       end
     end
 

data/lib/karafka/cli.rb

@@ -7,7 +7,7 @@ module Karafka
   #
   # @note Whole Cli is built using Thor
   # @see https://github.com/erikhuda/thor
-  class Cli
+  class Cli < Thor
     package_name 'Karafka'
 
     class << self

data/lib/karafka/cli/console.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli
+  class Cli < Thor
     # Console Karafka Cli action
     class Console < Base
       desc 'Start the Karafka console (short-cut alias: "c")'
@@ -22,7 +22,7 @@ module Karafka
       # Start the Karafka console
       def call
         cli.info
-        system self.class.command
+        exec self.class.command
       end
     end
   end

data/lib/karafka/cli/flow.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli
+  class Cli < Thor
     # Description of topics flow (incoming/outgoing)
     class Flow < Base
       desc 'Print application data flow (incoming => outgoing)'

data/lib/karafka/cli/info.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli
+  class Cli < Thor
     # Info Karafka Cli action
     class Info < Base
       desc 'Print configuration details and other options of your application'
@@ -14,7 +14,7 @@ module Karafka
         info = [
           "Karafka framework version: #{Karafka::VERSION}",
           "Application client id: #{config.client_id}",
-          "Inline processing: #{config.inline_processing}",
+          "Processing backend: #{config.processing_backend}",
           "Batch consuming: #{config.batch_consuming}",
           "Batch processing: #{config.batch_processing}",
           "Number of threads: #{config.concurrency}",

data/lib/karafka/cli/install.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli
+  class Cli < Thor
     # Install Karafka Cli action
     class Install < Base
       desc 'Install all required things for Karafka application in current directory'
@@ -20,7 +20,7 @@ module Karafka
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
-        'app.rb.example' => Karafka.boot_file.basename,
+        'karafka.rb.example' => Karafka.boot_file.basename,
         'sidekiq.yml.example' => 'config/sidekiq.yml.example',
         'application_worker.rb.example' => 'app/workers/application_worker.rb',
         'application_controller.rb.example' => 'app/controllers/application_controller.rb',

data/lib/karafka/cli/server.rb

@@ -2,27 +2,30 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli
+  class Cli < Thor
     # Server Karafka Cli action
     class Server < Base
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
       option :daemon, default: false, type: :boolean, aliases: :d
       option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
+      option :consumer_groups, type: :array, default: nil, aliases: :g
 
       # Start the Karafka server
       def call
+        validate!
+
         puts 'Starting Karafka server'
         cli.info
 
         if cli.options[:daemon]
+          FileUtils.mkdir_p File.dirname(cli.options[:pid])
           # For some reason Celluloid spins threads that break forking
           # Threads are not shutdown immediately so deamonization will stale until
           # those threads are killed by Celluloid manager (via timeout)
           # There's nothing initialized here yet, so instead we shutdown celluloid
           # and run it again when we need (after fork)
           Celluloid.shutdown
-          validate!
           daemonize
           Celluloid.boot
         end
@@ -30,17 +33,22 @@ module Karafka
         # Remove pidfile on shutdown
         ObjectSpace.define_finalizer(String.new, proc { send(:clean) })
 
+        # We assign active topics on a server level, as only server is expected to listen on
+        # part of the topics
+        Karafka::Server.consumer_groups = cli.options[:consumer_groups]
+
         # After we fork, we can boot celluloid again
         Karafka::Server.run
       end
 
       private
 
-      # Prepare (if not exists) directory for a pidfile and check if there is no running karafka
-      # instance already (and raise error if so)
+      # Checks the server cli configuration
+      # options validations in terms of app setup (topics, pid existence, etc)
       def validate!
-        FileUtils.mkdir_p File.dirname(cli.options[:pid])
-        raise "#{cli.options[:pid]} already exists" if File.exist?(cli.options[:pid])
+        result = Schemas::ServerCliOptions.call(cli.options)
+        return if result.success?
+        raise Errors::InvalidConfiguration, result.errors
       end
 
       # Detaches current process into background and writes its pidfile

data/lib/karafka/cli/worker.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli
+  class Cli < Thor
     # Worker Karafka Cli action
     class Worker < Base
       desc 'Start the Karafka Sidekiq worker (short-cut alias: "w")'

data/lib/karafka/connection/config_adapter.rb

@@ -41,13 +41,7 @@ module Karafka
         # @return [Hash] hash with all the settings required by Kafka#consumer method
         def consumer(consumer_group)
           settings = { group_id: consumer_group.id }
-
-          kafka_configs.each do |setting_name, setting_value|
-            next unless AttributesMap.config_adapter[:consumer].include?(setting_name)
-            next if settings.keys.include?(setting_name)
-            settings[setting_name] = setting_value
-          end
-
+          settings = fetch_for(:consumer, settings)
           sanitize(settings)
         end
 
@@ -57,15 +51,7 @@ module Karafka
         # @return [Hash] hash with all the settings required by
         #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
         def consuming(_consumer_group)
-          settings = {}
-
-          kafka_configs.each do |setting_name, setting_value|
-            next unless AttributesMap.config_adapter[:consuming].include?(setting_name)
-            next if settings.keys.include?(setting_name)
-            settings[setting_name] = setting_value
-          end
-
-          sanitize(settings)
+          sanitize(fetch_for(:consuming))
         end
 
         # Builds all the configuration settings for kafka consumer#subscribe method
@@ -73,18 +59,33 @@ module Karafka
         # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
         def subscription(topic)
           settings = { start_from_beginning: topic.start_from_beginning }
+          settings = fetch_for(:subscription, settings)
+          [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
+        end
+
+        # Builds all the configuration settings required by kafka consumer#pause method
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Hash] hash with all the settings required to pause kafka consumer
+        def pausing(consumer_group)
+          { timeout: consumer_group.pause_timeout }
+        end
 
+        private
+
+        # Fetches proper settings for a given map namespace
+        # @param namespace_key [Symbol] namespace from attributes map config adapter hash
+        # @param preexisting_settings [Hash] hash with some preexisting settings that might have
+        #   been loaded in a different way
+        def fetch_for(namespace_key, preexisting_settings = {})
           kafka_configs.each do |setting_name, setting_value|
-            next unless AttributesMap.config_adapter[:subscription].include?(setting_name)
-            next if settings.keys.include?(setting_name)
-            settings[setting_name] = setting_value
+            next unless AttributesMap.config_adapter[namespace_key].include?(setting_name)
+            next if preexisting_settings.keys.include?(setting_name)
+            preexisting_settings[setting_name] = setting_value
           end
 
-          [topic.name, sanitize(settings)]
+          preexisting_settings
         end
 
-        private
-
         # Removes nil containing keys from the final settings so it can use Kafkas driver
         #   defaults for those
         # @param settings [Hash] settings that may contain nil values

data/lib/karafka/connection/messages_consumer.rb

@@ -20,9 +20,19 @@ module Karafka
       def fetch_loop
         send(
           consumer_group.batch_consuming ? :consume_each_batch : :consume_each_message
-        ) do |messages|
-          yield(messages)
-        end
+        ) { |messages| yield(messages) }
+      rescue Kafka::ProcessingError => e
+        # If there was an error during processing, we have to log it, pause current partition
+        # and process other things
+        Karafka.monitor.notice_error(self.class, e.cause)
+        pause(e.topic, e.partition)
+        retry
+        # This is on purpose - see the notes for this method
+        # rubocop:disable RescueException
+      rescue Exception => e
+        # rubocop:enable RescueException
+        Karafka.monitor.notice_error(self.class, e)
+        retry
       end
 
       # Gracefuly stops topic consumption
@@ -35,6 +45,16 @@ module Karafka
 
       attr_reader :consumer_group
 
+      # Pauses processing of a given topic partition
+      # @param topic [String] topic that we want to pause
+      # @param partition [Integer] number partition that we want to pause
+      def pause(topic, partition)
+        settings = ConfigAdapter.pausing(consumer_group)
+        return false unless settings[:timeout].positive?
+        kafka_consumer.pause(topic, partition, settings)
+        true
+      end
+
       # Consumes messages from Kafka in batches
       # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
       def consume_each_batch

data/lib/karafka/connection/messages_processor.rb

@@ -24,48 +24,29 @@ module Karafka
           controller = Karafka::Routing::Router.build("#{group_id}_#{mapped_topic}")
           handler = controller.topic.batch_processing ? :process_batch : :process_each
 
-          send(handler, controller, mapped_topic, kafka_messages)
-          # This is on purpose - see the notes for this method
-          # rubocop:disable RescueException
-        rescue Exception => e
-          # rubocop:enable RescueException
-          Karafka.monitor.notice_error(self, e)
+          send(handler, controller, kafka_messages)
         end
 
         private
 
         # Processes whole batch in one request (all at once)
         # @param controller [Karafka::BaseController] base controller descendant
-        # @param mapped_topic [String] mapped topic name
         # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
-        def process_batch(controller, mapped_topic, kafka_messages)
-          messages_batch = kafka_messages.map do |kafka_message|
-            # Since we support topic mapping (for Kafka providers that require namespaces)
-            # we have to overwrite topic with our mapped topic version
-            # @note For the default mapper, it will be the same as topic
-            # @note We have to use instance_variable_set, as the Kafka::FetchedMessage does not
-            #   provide attribute writers
-            kafka_message.instance_variable_set(:'@topic', mapped_topic)
-            kafka_message
-          end
-
-          controller.params_batch = messages_batch
-
-          Karafka.monitor.notice(self, messages_batch)
-
+        def process_batch(controller, kafka_messages)
+          controller.params_batch = kafka_messages
+          Karafka.monitor.notice(self, kafka_messages)
           controller.schedule
         end
 
         # Processes messages one by one (like with std http requests)
         # @param controller [Karafka::BaseController] base controller descendant
-        # @param mapped_topic [String] mapped topic name
         # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
-        def process_each(controller, mapped_topic, kafka_messages)
+        def process_each(controller, kafka_messages)
           kafka_messages.each do |kafka_message|
             # @note This is a simple trick - we just process one after another, but in order
             # not to handle everywhere both cases (single vs batch), we just "fake" batching with
             # a single message for each
-            process_batch(controller, mapped_topic, [kafka_message])
+            process_batch(controller, [kafka_message])
           end
         end
       end

data/lib/karafka/errors.rb

@@ -41,5 +41,9 @@ module Karafka
     # Raised when processing messages in batches but still want to use #params instead of
     # #params_batch
     ParamsMethodUnavailable = Class.new(BaseError)
+
+    # Raised when for some reason we try to use invalid processing backend and
+    # we bypass validations
+    InvalidProcessingBackend = Class.new(BaseError)
   end
 end

data/lib/karafka/fetcher.rb

@@ -26,7 +26,7 @@ module Karafka
 
     # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
     def listeners
-      @listeners ||= App.consumer_groups.map do |consumer_group|
+      @listeners ||= App.consumer_groups.active.map do |consumer_group|
         Karafka::Connection::Listener.new(consumer_group)
       end
     end

data/lib/karafka/loader.rb

@@ -2,78 +2,28 @@
 
 module Karafka
   # Loader for requiring all the files in a proper order
-  # Some files needs to be required before other, so it will
-  # try to figure it out. It will load 'base*' files first and then
-  # any other.
-  class Loader
+  module Loader
     # Order in which we want to load app files
     DIRS = %w[
       config/initializers
       lib
-      app/helpers
-      app/inputs
-      app/decorators
-      app/models/concerns
-      app/models
-      app/responders
-      app/services
-      app/presenters
-      app/workers
-      app/controllers
-      app/aspects
       app
     ].freeze
 
     # Will load files in a proper order (based on DIRS)
     # @param [String] root path from which we want to start
-    def load(root)
+    def self.load(root)
       DIRS.each do |dir|
         path = File.join(root, dir)
+        next unless File.exist?(path)
         load!(path)
       end
     end
 
     # Requires all the ruby files from one path in a proper order
     # @param path [String] path (dir) from which we want to load ruby files in a proper order
-    # @note First we load all the base files that might be used in inheritance
-    def load!(path)
-      base_load!(path)
-      files_load!(path)
-    end
-
-    # Requires all the ruby files from one relative path inside application directory
-    # @param relative_path [String] relative path (dir) to a file inside application directory
-    #   from which we want to load ruby files in a proper order
-    def relative_load!(relative_path)
-      path = File.join(::Karafka.root, relative_path)
-      load!(path)
-    end
-
-    private
-
-    # Loads all the base files
-    # @param path [String] path (dir) from which we want to load ruby base files in a proper order
-    def base_load!(path)
-      bases = File.join(path, '**/base*.rb')
-      Dir[bases].sort(&method(:base_sorter)).each(&method(:require))
-    end
-
-    # Loads all other files (not base)
-    # @param path [String] path (dir) from which we want to load ruby files in a proper order
-    # @note Technically it will load the base files again but they are already loaded so nothing
-    #   will happen
-    def files_load!(path)
-      files = File.join(path, '**/*.rb')
-      Dir[files].sort.each(&method(:require))
-    end
-
-    # @return [Integer] order for sorting
-    # @note We need sort all base files based on their position in a file tree
-    #   so all the files that are "higher" should be loaded first
-    # @param str1 [String] first string for comparison
-    # @param str2 [String] second string for comparison
-    def base_sorter(str1, str2)
-      str1.count('/') <=> str2.count('/')
+    def self.load!(path)
+      require_all(path)
     end
   end
 end

data/lib/karafka/monitor.rb

@@ -55,7 +55,7 @@ module Karafka
     def caller_exceptions_map
       @caller_exceptions_map ||= {
         error: [
-          Karafka::Connection::MessagesProcessor,
+          Karafka::Connection::MessagesConsumer,
           Karafka::Connection::Listener,
           Karafka::Params::Params
         ],

data/lib/karafka/params/params.rb

@@ -10,7 +10,6 @@ module Karafka
     class Params < HashWithIndifferentAccess
       # Kafka::FetchedMessage attributes that we want to use inside of params
       KAFKA_MESSAGE_ATTRIBUTES = %i[
-        topic
         value
         partition
         offset
@@ -44,6 +43,11 @@ module Karafka
               KAFKA_MESSAGE_ATTRIBUTES.each do |attribute|
                 instance[attribute] = message.send(attribute)
               end
+
+              # When we get raw messages, they might have a topic, that was modified by a
+              # topic mapper. We need to "reverse" this change and map back to the non-modified
+              # format, so our internal flow is not corrupted with the mapping
+              instance[:topic] = Karafka::App.config.topic_mapper.incoming(message.topic)
             end
           end
         end

data/lib/karafka/parsers/json.rb

@@ -10,8 +10,8 @@ module Karafka
       # @example
       #   Json.parse("{\"a\":1}") #=> { 'a' => 1 }
       def self.parse(content)
-        ::Yajl::Parser.parse(content)
-      rescue ::Yajl::ParseError => e
+        ::MultiJson.load(content)
+      rescue ::MultiJson::ParseError => e
         raise ::Karafka::Errors::ParserError, e
       end
 

data/lib/karafka/routing/builder.rb

@@ -32,6 +32,13 @@ module Karafka
         end
       end
 
+      # @return [Array<Karafka::Routing::ConsumerGroup>] only active consumer groups that
+      #   we want to use. Since Karafka supports multi-process setup, we need to be able
+      #   to pick only those consumer groups that should be active in our given process context
+      def active
+        select(&:active?)
+      end
+
       private
 
       # Builds and saves given consumer group

data/lib/karafka/routing/consumer_group.rb

@@ -10,16 +10,23 @@ module Karafka
 
       attr_reader :topics
       attr_reader :id
+      attr_reader :name
 
-      # @param id [String, Symbol] raw id of this consumer group. Raw means, that it does not
+      # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
       #   We add it to make a multi-system development easier for people that don't use
       #   kafka and don't understand the concept of consumer groups.
-      def initialize(id)
-        @id = "#{Karafka::App.config.client_id.to_s.underscore}_#{id}"
+      def initialize(name)
+        @name = name
+        @id = "#{Karafka::App.config.client_id.to_s.underscore}_#{@name}"
         @topics = []
       end
 
+      # @return [Boolean] true if this consumer group should be active in our current process
+      def active?
+        Karafka::Server.consumer_groups.include?(name)
+      end
+
       # Builds a topic representation inside of a current consumer group route
       # @param name [String, Symbol] name of topic to which we want to subscribe
       # @yield Evaluates a given block in a topic context

data/lib/karafka/routing/topic.rb

@@ -36,7 +36,7 @@ module Karafka
       #   background job
       # @note If not provided - will be built based on the provided controller
       def worker
-        @worker ||= inline_processing ? nil : Karafka::Workers::Builder.new(controller).build
+        @worker ||= processing_backend == :sidekiq ? Workers::Builder.new(controller).build : nil
       end
 
       # @return [Class, nil] Class (not an instance) of a responder that should respond from

data/lib/karafka/schemas/config.rb

@@ -20,11 +20,11 @@ module Karafka
         end
       end
 
-      optional(:inline_processing).filled(:bool?)
+      optional(:processing_backend).filled(included_in?: %i[inline sidekiq])
 
-      # If inline_processing is true, redis should be filled
-      rule(redis_presence: %i[redis inline_processing]) do |redis, inline_processing|
-        inline_processing.false?.then(redis.filled?)
+      # If we want to use sidekiq, then redis needs to be configured
+      rule(redis_presence: %i[redis processing_backend]) do |redis, processing_backend|
+        processing_backend.eql?(:sidekiq).then(redis.filled?)
       end
 
       optional(:connection_pool).schema do

data/lib/karafka/schemas/consumer_group.rb

@@ -6,7 +6,7 @@ module Karafka
     ConsumerGroupTopic = Dry::Validation.Schema do
       required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
       required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:inline_processing).filled(:bool?)
+      required(:processing_backend).filled(included_in?: %i[inline sidekiq])
       required(:controller).filled
       required(:parser).filled
       required(:interchanger).filled
@@ -20,11 +20,11 @@ module Karafka
       required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
       required(:seed_brokers).filled(:array?)
       required(:session_timeout).filled(:int?)
+      required(:pause_timeout).filled(:int?, gteq?: 0)
       required(:offset_commit_interval).filled(:int?)
       required(:offset_commit_threshold).filled(:int?)
       required(:offset_retention_time) { none?.not > int? }
       required(:heartbeat_interval).filled(:int?, gteq?: 0)
-      required(:topic_mapper).filled
       required(:connect_timeout).filled(:int?, gt?: 0)
       required(:socket_timeout).filled(:int?, gt?: 0)
       required(:max_wait_time).filled(:int?, gteq?: 0)

data/lib/karafka/schemas/server_cli_options.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Schemas
+    # Schema for validating correctness of the server cli command options
+    # We validate some basics + the list of consumer_groups on which we want to use, to make
+    # sure that all of them are defined, plus that a pidfile does not exist
+    ServerCliOptions = Dry::Validation.Schema do
+      configure do
+        option :consumer_groups
+
+        def self.messages
+          super.merge(
+            en: {
+              errors: {
+                consumer_groups_inclusion: 'Unknown consumer group.',
+                pid_existence: 'Pidfile already exists.'
+              }
+            }
+          )
+        end
+      end
+
+      optional(:pid).filled(:str?)
+      optional(:daemon).filled(:bool?)
+      optional(:consumer_groups).filled(:array?)
+
+      validate(consumer_groups_inclusion: :consumer_groups) do |consumer_groups|
+        # If there were no consumer_groups declared in the server cli, it means that we will
+        # run all of them and no need to validate them here at all
+        if consumer_groups.nil?
+          true
+        else
+          (consumer_groups - Karafka::Routing::Builder.instance.map(&:name)).empty?
+        end
+      end
+
+      validate(pid_existence: :pid) do |pid|
+        pid ? !File.exist?(pid) : true
+      end
+    end
+  end
+end

data/lib/karafka/server.rb

@@ -8,6 +8,9 @@ module Karafka
       # So we can have access to them later on and be able to stop them on exit
       attr_reader :consumers
 
+      # Writer for list of consumer groups that we want to consume in our current process context
+      attr_writer :consumer_groups
+
       # Method which runs app
       def run
         @consumers = Concurrent::Array.new
@@ -17,6 +20,13 @@ module Karafka
         start_supervised
       end
 
+      # @return [Array<String>] array with names of consumer groups that should be consumed in a
+      #   current server context
+      def consumer_groups
+        # If not specified, a server will listed on all the topics
+        @consumer_groups ||= Karafka::App.consumer_groups.map(&:name).freeze
+      end
+
       private
 
       # @return [Karafka::Process] process wrapper instance used to catch system signal calls

data/lib/karafka/setup/config.rb

@@ -18,9 +18,9 @@ module Karafka
       # option client_id [String] kafka client_id - used to provide
       #   default Kafka groups namespaces and identify that app in kafka
       setting :client_id
-      # If inline_processing is set to true, we won't enqueue jobs, instead we will run them
-      # immediately
-      setting :inline_processing, false
+      # How should we process messages. For now we support inline mode (asap in the process) or
+      # sidekiq mode (schedule to sidekiq)
+      setting :processing_backend, :inline
       # option logger [Instance] logger that we want to use
       setting :logger, -> { ::Karafka::Logger.instance }
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
@@ -51,7 +51,7 @@ module Karafka
         # or the opposite for bigger systems
         setting :size, lambda {
           [
-            ::Karafka::App.consumer_groups.count,
+            ::Karafka::App.consumer_groups.active.count,
             Sidekiq.options[:concurrency]
           ].max
         }
@@ -67,6 +67,11 @@ module Karafka
         # option session_timeout [Integer] the number of seconds after which, if a client
         #   hasn't contacted the Kafka cluster, it will be kicked out of the group.
         setting :session_timeout, 30
+        # Time that a given partition will be paused from processing messages, when message
+        # processing fails. It allows us to process other partitions, while the error is being
+        # resolved and also "slows" things down, so it prevents from "eating" up all messages and
+        # processing them with failed code
+        setting :pause_timeout, 10
         # option offset_commit_interval [Integer] the interval between offset commits,
         #   in seconds.
         setting :offset_commit_interval, 10

data/lib/karafka/templates/application_controller.rb.example

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Application controller from which all Karafka controllers should inherit
 # You can rename it if it would conflict with your current code base (in case you're integrating
 # Karafka with other frameworks)

data/lib/karafka/templates/application_responder.rb.example

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Application responder from which all Karafka responders should inherit
 # You can rename it if it would conflict with your current code base (in case you're integrating
 # Karafka with other frameworks)

data/lib/karafka/templates/application_worker.rb.example

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Application worker from which all workers should inherit
 # You can rename it if it would conflict with your current code base (in case you're integrating
 # Karafka with other frameworks). Karafka will use first direct descendant of Karafka::BaseWorker

data/lib/karafka/templates/{app.rb.example → karafka.rb.example}

@@ -1,19 +1,26 @@
+# frozen_string_literal: true
+
+# Non Ruby on Rails setup
 ENV['RACK_ENV'] ||= 'development'
 ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
-
 Bundler.require(:default, ENV['KARAFKA_ENV'])
+Karafka::Loader.load(Karafka::App.root)
 
-Karafka::Loader.new.load(Karafka::App.root)
+# Ruby on Rails setup
+# Remove whole non-Rails setup that is above and uncomment the 4 lines below
+# ENV['RAILS_ENV'] ||= 'development'
+# ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
+# require ::File.expand_path('../config/environment', __FILE__)
+# Rails.application.eager_load!
 
-# App class
-class App < Karafka::App
+class KarafkaApp < Karafka::App
   setup do |config|
     config.kafka.seed_brokers = %w( 127.0.0.1:9092 )
     config.client_id = 'example_app'
     config.redis = {
       url: 'redis://localhost:6379'
     }
-    config.inline_processing = Karafka.env.development?
+    config.processing_backend = Karafka.env.development? ? :inline : :sidekiq
     config.batch_consuming = true
   end
 
@@ -35,4 +42,4 @@ class App < Karafka::App
   end
 end
 
-App.boot!
+KarafkaApp.boot!

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '0.6.0.rc1'
+  VERSION = '0.6.0.rc2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 0.6.0.rc1
+  version: 0.6.0.rc2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-15 00:00:00.000000000 Z
+date: 2017-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
@@ -153,75 +153,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.7'
 - !ruby/object:Gem::Dependency
-  name: yajl-ruby
+  name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '1.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '1.12'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: require_all
   requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.6'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.6'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.14'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.14'
-- !ruby/object:Gem::Dependency
-  name: byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '1.4'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.4'
 description: Framework used to simplify Apache Kafka based Ruby applications development
 email:
 - maciej@coditsu.io
@@ -292,6 +250,7 @@ files:
 - lib/karafka/schemas/config.rb
 - lib/karafka/schemas/consumer_group.rb
 - lib/karafka/schemas/responder_usage.rb
+- lib/karafka/schemas/server_cli_options.rb
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb
 - lib/karafka/setup/configurators/base.rb
@@ -299,10 +258,10 @@ files:
 - lib/karafka/setup/configurators/sidekiq.rb
 - lib/karafka/setup/configurators/water_drop.rb
 - lib/karafka/status.rb
-- lib/karafka/templates/app.rb.example
 - lib/karafka/templates/application_controller.rb.example
 - lib/karafka/templates/application_responder.rb.example
 - lib/karafka/templates/application_worker.rb.example
+- lib/karafka/templates/karafka.rb.example
 - lib/karafka/templates/sidekiq.yml.example
 - lib/karafka/version.rb
 - lib/karafka/workers/builder.rb