karafka 0.5.0.2 → 0.5.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b15329e4a83277e01ced5cb6e7e0419d392707a6
-  data.tar.gz: 6080d3348e67cd407e37472e97a400f8d648bdb0
+  metadata.gz: 6d336fa4ee7c6d7a9b38ee1ea08e70a9f6208312
+  data.tar.gz: 219b4f431dc21cbe403f2f8aa50b81eae850c93c
 SHA512:
-  metadata.gz: 109523139cb10a65641300fec95edd09b55a190ae912df450d614b016c2e76988511dbcd29510400bc7d0ce3fac139035736628dc5347e2919b927f7e0284e2d
-  data.tar.gz: ba2958568a92b148c956355b164f35ec6408f35a2efe55dcb880ac8fd8009acb951ed89904d71b2341ee66304def6f125b10aa4536a620c849512432d9c94e50
+  metadata.gz: 288e9e7054f1716c9d2600fb572b4fd288c6d33a29df89b644d584fb91d410da40ace8628889d7d936bd0526b40f0ffcf75729a41826c1bd0b6ddd1e5262e5d1
+  data.tar.gz: 00214b9cf81f378b42d9a3c78151f1a8f95cfe5a8110c7566ae2280edf41bbcd9cc527e586d7d3978195418c1b8d6e48821a7ac21855c1578ca193e78c484241

data/.ruby-version

@@ -1 +1 @@
-2.4.0
+2.4.1

data/.travis.yml

@@ -5,5 +5,6 @@ rvm:
   - 2.3.2
   - 2.3.3
   - 2.4.0
+  - 2.4.1
 script:
   - bundle exec rake

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Karafka framework changelog
 
+## 0.5.0.3
+- #132 - When Kafka is gone, should reconnect after a time period
+- #136 - new ruby-kafka version + other gem bumps
+- ruby-kafka update
+- #135 - NonMatchingRouteError - better error description in the code
+- #140 - Move Capistrano Karafka to a different specific gem
+- #110 - Add call method on a responder class to alias instance build and call
+- #76 - Configs validator
+- #138 - Possibility to have no worker class defined if inline_mode is being used
+- #145 - Topic Mapper
+- Ruby update to 2.4.1
+- Gem bump x2
+- #158 - Update docs section on heroku usage
+- #150 - Add support for start_from_beginning on a per topic basis
+- #148 - Lower Karafka Sidekiq dependency
+- Allow karafka root to be specified from ENV
+- Handle SIGTERM as a shutdown command for kafka server to support Heroku deployment
+
 ## 0.5.0.2
 - Gems update x3
 - Default Ruby set to 2.3.3
@@ -16,7 +34,7 @@
 - #130 - start_from_beginning flag on routes and default
 - #128 - Monitor caller_label not working with super on inheritance
 - Renamed *inline* to *inline_mode* to stay consistent with flags that change the way karafka works (#125)
-- Dry-configurable dump to 0.5 with fixed proc value evaluation on retrieve patch (internal change)
+- Dry-configurable bump to 0.5 with fixed proc value evaluation on retrieve patch (internal change)
 
 ## 0.5.0.1
 - Fixed inconsistency in responders non-required topic definition. Now only required: false available
@@ -50,7 +68,7 @@
 - Using App name as a Kafka client_id
 - Automatic Capistrano integration
 - Responders support for handling better responses pipelining and better responses flow description and design (see README for more details)
-- Gem dump
+- Gem bump
 - Readme updates
 - karafka flow CLI command for printing the application flow
 - Some internal refactorings
@@ -59,7 +77,7 @@
 - #87 - Reconsume mode with crone for better Rails/Rack integration
 - Moved Karafka server related stuff into separate Karafka::Server class
 - Renamed Karafka::Runner into Karafka::Fetcher
-- Gem dump
+- Gem bump
 - Added chroot option to Zookeeper options
 - Moved BROKERS_PATH into config from constant
 - Added Karafka consume CLI action for a short running single consumption round
@@ -72,7 +90,7 @@
 - #63 - Graceful shutdown with current offset state during data processing
 - #65 - Example of NewRelic monitor is outdated
 - #71 - Setup should be executed after user code is loaded
-- Gem dump x3
+- Gem bump x3
 - Rubocop remarks
 - worker_timeout config option has been removed. It now needs to be defined manually by the framework user because WorkerGlass::Timeout can be disabled and we cannot use Karafka settings on a class level to initialize user code stuff
 - Moved setup logic under setup/Setup namespace
@@ -82,12 +100,12 @@
 - #81 - Switch config management to dry configurable
 - Version fix
 - Dropped support for Ruby 2.1.*
-- Ruby dump to 2.3.1
+- Ruby bump to 2.3.1
 
 ## 0.4.0
 - Added WaterDrop gem with default configuration
 - Refactoring of config logic to simplify adding new dependencies that need to be configured based on #setup data
-- Gem dump
+- Gem bump
 - Readme updates
 - Renamed cluster to actor_cluster for method names
 - Replaced SidekiqGlass with generic WorkerGlass lib
@@ -128,21 +146,21 @@
 
 ## 0.3.0
 - Switched from custom ParserError for each parser to general catching of Karafka::Errors::ParseError and its descendants
-- Gem dump
+- Gem bump
 - Fixed #32 - now when using custom workers that does not inherit from Karafka::BaseWorker perform method is not required. Using custom workers means that the logic that would normally lie under #perform, needs to be executed directly from the worker.
 - Fixed #31 - Technically didn't fix because this is how Sidekiq is meant to work, but provided possibility to assign custom interchangers that allow to bypass JSON encoding issues by converting data that goes to Redis to a required format (and parsing it back when it is fetched)
 - Added full parameters lazy load - content is no longer loaded during #perform_async if params are not used in before_enqueue
 - No more namespaces for Redis by default (use separate DBs)
 
 ## 0.1.21
-- Sidekiq 4.0.1 dump
-- Gem dump
+- Sidekiq 4.0.1 bump
+- Gem bump
 - Added direct celluloid requirement to Karafka (removed from Sidekiq)
 
 ## 0.1.19
 - Internal call - schedule naming change
 - Enqueue to perform_async naming in controller to follow Sidekiqs naming convention
-- Gem dump
+- Gem bump
 
 ## 0.1.18
 - Changed Redis configuration options into a single hash that is directly passed to Redis setup for Sidekiq
@@ -174,7 +192,7 @@
 
 ## 0.1.13
 - Ability to assign custom workers and use them bypassing Karafka::BaseWorker (or its descendants)
-- Gem dump
+- Gem bump
 
 ## 0.1.12
 - All internal errors went to Karafka::Errors namespace

data/Gemfile.lock

@@ -1,22 +1,23 @@
 PATH
   remote: .
   specs:
-    karafka (0.5.0.2)
+    karafka (0.5.0.3)
       activesupport (~> 5.0)
       celluloid (~> 0.17)
-      dry-configurable (~> 0.5)
+      dry-configurable (~> 0.7)
+      dry-validation (~> 0.10.6)
       envlogic (~> 1.0)
       rake (~> 11.3)
-      ruby-kafka (= 0.3.15)
-      sidekiq (~> 4.2)
+      ruby-kafka (= 0.3.17)
+      sidekiq (>= 4.2)
       thor (~> 0.19)
-      waterdrop (~> 0.3.2.1)
+      waterdrop (~> 0.3.2.4)
       worker-glass (~> 0.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.0.1)
+    activesupport (5.0.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
@@ -54,14 +55,39 @@ GEM
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
     colorize (0.8.1)
-    concurrent-ruby (1.0.4)
+    concurrent-ruby (1.0.5)
     connection_pool (2.2.1)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.2.5)
     docile (1.1.5)
-    dry-configurable (0.5.0)
+    dry-configurable (0.7.0)
       concurrent-ruby (~> 1.0)
+    dry-container (0.6.0)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.1, >= 0.1.3)
+    dry-core (0.3.1)
+      concurrent-ruby (~> 1.0)
+    dry-equalizer (0.2.0)
+    dry-logic (0.4.1)
+      dry-container (~> 0.2, >= 0.2.6)
+      dry-core (~> 0.2)
+      dry-equalizer (~> 0.2)
+    dry-types (0.10.3)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.1)
+      dry-container (~> 0.3)
+      dry-core (~> 0.2, >= 0.2.1)
+      dry-equalizer (~> 0.2)
+      dry-logic (~> 0.4, >= 0.4.0)
+      inflecto (~> 0.0.0, >= 0.0.2)
+    dry-validation (0.10.7)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.1, >= 0.1.3)
+      dry-core (~> 0.2, >= 0.2.1)
+      dry-equalizer (~> 0.2)
+      dry-logic (~> 0.4, >= 0.4.0)
+      dry-types (~> 0.9, >= 0.9.0)
     envlogic (1.0.3)
       activesupport
     equalizer (0.0.11)
@@ -84,15 +110,16 @@ GEM
       rake (>= 10, < 12)
       rubocop (>= 0.36.0)
       sysexits (~> 1.1)
-    hitimes (1.2.4)
+    hitimes (1.2.5)
     i18n (0.7.0)
     ice_nine (0.11.2)
+    inflecto (0.0.2)
     json (2.0.2)
     launchy (2.4.3)
       addressable (~> 2.3)
     method_source (0.8.2)
     minitest (5.10.1)
-    null-logger (0.1.3)
+    null-logger (0.1.4)
     parser (2.3.1.2)
       ast (~> 2.2)
     path_expander (1.0.0)
@@ -115,8 +142,8 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    rack (2.0.1)
-    rack-protection (1.5.3)
+    rack (2.0.3)
+    rack-protection (2.0.0)
       rack
     rainbow (2.1.0)
     rake (11.3.0)
@@ -146,7 +173,7 @@ GEM
       unicode-display_width (~> 1.0, >= 1.0.1)
     rubocop-rspec (1.5.3)
       rubocop (>= 0.42.0)
-    ruby-kafka (0.3.15)
+    ruby-kafka (0.3.17)
     ruby-progressbar (1.8.1)
     ruby_parser (3.8.2)
       sexp_processor (~> 4.1)
@@ -166,11 +193,11 @@ GEM
     shoulda-context (1.2.1)
     shoulda-matchers (2.8.0)
       activesupport (>= 3.0.0)
-    sidekiq (4.2.9)
+    sidekiq (5.0.3)
       concurrent-ruby (~> 1.0)
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
-      redis (~> 3.2, >= 3.2.1)
+      redis (~> 3.3, >= 3.3.3)
     simplecov (0.12.0)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
@@ -184,7 +211,7 @@ GEM
     timecop (0.8.1)
     timers (4.1.2)
       hitimes
-    tzinfo (1.2.2)
+    tzinfo (1.2.3)
       thread_safe (~> 0.1)
     unicode-display_width (1.1.0)
     virtus (1.0.5)
@@ -192,14 +219,14 @@ GEM
       coercible (~> 1.0)
       descendants_tracker (~> 0.0, >= 0.0.3)
       equalizer (~> 0.0, >= 0.0.9)
-    waterdrop (0.3.2.2)
+    waterdrop (0.3.2.4)
       bundler
       connection_pool
-      dry-configurable (~> 0.5)
+      dry-configurable (~> 0.6)
       null-logger
       rake
       ruby-kafka
-    worker-glass (0.2.2)
+    worker-glass (0.2.3)
       activesupport
       null-logger
     yard (0.9.5)
@@ -214,4 +241,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   1.13.7
+   1.14.6

data/README.md

@@ -22,6 +22,7 @@ Karafka not only handles incoming messages but also provides tools for building
     - [Configurators](#configurators)
     - [Environment variables settings](#environment-variables-settings)
     - [Kafka brokers auto-discovery](#kafka-brokers-auto-discovery)
+    - [Topic mappers](#topic-mappers)
   - [Usage](#usage)
     - [Karafka CLI](#karafka-cli)
     - [Routing](#routing)
@@ -53,6 +54,7 @@ Karafka not only handles incoming messages but also provides tools for building
   - [Deployment](#deployment)
       - [Capistrano](#capistrano)
       - [Docker](#docker)
+      - [Heroku](#heroku)
   - [Sidekiq Web UI](#sidekiq-web-ui)
   - [Concurrency](#concurrency)
   - [Integrating with other frameworks](#integrating-with-other-frameworks)
@@ -112,13 +114,14 @@ Karafka has following configuration options:
 | Option                        | Required | Value type        | Description                                                                                                |
 |-------------------------------|----------|-------------------|------------------------------------------------------------------------------------------------------------|
 | name                          | true     | String            | Application name                                                                                           |
-| redis                         | true     | Hash              | Hash with Redis configuration options                                                                      |
+| topic_mapper                  | false    | Class/Module      | Mapper for hiding Kafka provider specific topic prefixes/postfixes, so internaly we use "pure" topics      |
+| redis                         | false    | Hash              | Hash with Redis configuration options. It is required if inline_mode is off.                               |
 | inline_mode                   | false    | Boolean           | Do we want to perform logic without enqueuing it with Sidekiq (directly and asap)                          |
 | batch_mode                    | false    | Boolean           | Should the incoming messages be consumed in batches, or one at a time                                      |
 | start_from_beginning          | false    | Boolean           | Consume messages starting at the beginning or consume new messages that are produced at first run          |
 | monitor                       | false    | Object            | Monitor instance (defaults to Karafka::Monitor)                                                            |
 | logger                        | false    | Object            | Logger instance (defaults to Karafka::Logger)                                                              |
-| kafka.hosts                   | false    | Array<String>     | Kafka server hosts. If 1 provided, Karafka will discover cluster structure automatically                   |
+| kafka.hosts                   | true     | Array<String>     | Kafka server hosts. If 1 provided, Karafka will discover cluster structure automatically                   |
 | kafka.session_timeout         | false    | Integer           | The number of seconds after which, if a consumer hasn't contacted the Kafka cluster, it will be kicked out |
 | kafka.offset_commit_interval  | false    | Integer           | The interval between offset commits in seconds                                                             |
 | kafka.offset_commit_threshold | false    | Integer           | The number of messages that can be processed before their offsets are committed                            |
@@ -173,11 +176,56 @@ There are several env settings you can use:
 |-------------------|-----------------|-------------------------------------------------------------------------------|
 | KARAFKA_ENV       | development     | In what mode this application should boot (production/development/test/etc)   |
 | KARAFKA_BOOT_FILE | app_root/app.rb | Path to a file that contains Karafka app configuration and booting procedures |
+| KARAFKA_ROOT_DIR  | Gemfile location| Path to Karafka's root directory                                              |
 
 ### Kafka brokers auto-discovery
 
 Karafka supports Kafka brokers auto-discovery during startup and on failures. You need to provide at least one Kafka broker, from which the entire Kafka cluster will be discovered. Karafka will refresh list of available brokers if something goes wrong. This allows it to be aware of changes that happen in the infrastructure (adding and removing nodes).
 
+### Topic mappers
+
+Some Kafka cloud providers require topics to be namespaced with user name. This approach is understandable, but at the same time, makes your applications less provider agnostic. To target that issue, you can create your own topic mapper that will sanitize incoming/outgoing topic names, so your logic won't be binded to those specific versions of topic names.
+
+Mapper needs to implement two following methods:
+
+  - ```#incoming``` - accepts an incoming "namespace dirty" version ot topic. Needs to return sanitized topic.
+  - ```#outgoing``` - accepts outgoing sanitized topic version. Needs to return namespaced one.
+
+Given each of the topics needs to have "karafka." prefix, your mapper could look like that:
+
+```ruby
+class KarafkaTopicMapper
+  def initialize(prefix)
+    @prefix = prefix
+  end
+
+  def incoming(topic)
+    topic.to_s.gsub("#{@prefix}.", '')
+  end
+
+  def outgoing(topic)
+    "#{@prefix}.#{topic}"
+  end
+end
+
+mapper = KarafkaTopicMapper.new('karafka')
+mapper.incoming('karafka.my_super_topic') #=> 'my_super_topic'
+mapper.outgoing('my_other_topic') #=> 'karafka.my_other_topic'
+```
+
+To use custom mapper, just assign it during application configuration:
+
+```ruby
+class App < Karafka::App
+  setup do |config|
+    # Other settings
+    config.topic_mapper = MyCustomMapper.new('username')
+  end
+end
+```
+
+Topic mapper automatically integrates with both messages consumer and responders.
+
 ## Usage
 
 ### Karafka CLI
@@ -717,7 +765,7 @@ Here's a simple example of monitor that is used to handle errors logging into Ai
 class AppMonitor < Karafka::Monitor
   def notice_error(caller_class, e)
     super
-    Airbrake.notify_or_ignore(e)
+    Airbrake.notify(e)
   end
 end
 ```
@@ -794,19 +842,7 @@ Since the only thing that is long-running is Karafka server, it should't be hard
 
 ### Capistrano
 
-Use the built-in Capistrano recipe for easy Karafka server start/stop and restart with deploys.
-
-In your **Capfile** file:
-
-```ruby
-require 'karafka/capistrano'
-```
-
-Take a look at the [load:defaults task](https://github.com/karafka/karafka/blob/master/lib/karafka/capistrano/karafka.cap) (top of file) for options you can set. For example, to specify a different pidfile than default:
-
-```ruby
-set :karafka_pid, ->{ File.join(shared_path, 'tmp', 'pids', 'karafka0') }
-```
+For details about integration with Capistrano, please go to [capistrano-karafka](https://github.com/karafka/capistrano-karafka) gem page.
 
 ### Docker
 
@@ -817,6 +853,37 @@ ENV KARAFKA_ENV production
 CMD bundle exec karafka server
 ```
 
+### Heroku
+
+Karafka may be deployed on [Heroku](https://www.heroku.com/), and works with
+[Heroku Kafka](https://www.heroku.com/kafka) and [Heroku Redis](https://www.heroku.com/redis).
+
+Set `KARAFKA_ENV`:
+```bash
+heroku config:set KARAFKA_ENV=production
+```
+
+Configure Karafka to use the Kafka and Redis configuration provided by Heroku:
+```ruby
+# app_root/app.rb
+class App < Karafka::App
+  setup do |config|
+    config.kafka.hosts = ENV['KAFKA_URL'].split(',') # Convert CSV list of broker urls to an array
+    config.kafka.ssl.ca_cert = ENV['KAFKA_TRUSTED_CERT'] if ENV['KAFKA_TRUSTED_CERT']
+    config.kafka.ssl.client_cert = ENV['KAFKA_CLIENT_CERT'] if ENV['KAFKA_CLIENT_CERT']
+    config.kafka.ssl.client_cert_key = ENV['KAFKA_CLIENT_CERT_KEY'] if ENV['KAFKA_CLIENT_CERT_KEY']
+    config.redis = { url: ENV['REDIS_URL'] }
+    # ...other configuration options...
+  end  
+end
+```
+
+Create your Procfile:
+```text
+karafka_server: bundle exec karafka server
+karafka_worker: bundle exec karafka worker
+```
+
 ## Sidekiq Web UI
 
 Karafka comes with a Sidekiq Web UI application that can display the current state of a Sidekiq installation. If you installed Karafka based on the install instructions, you will have a **config.ru** file that allows you to run standalone Puma instance with a Sidekiq Web UI.
@@ -863,6 +930,8 @@ ENV['RACK_ENV'] ||= 'development'
 ENV['KARAFKA_ENV'] = ENV['RACK_ENV']
 
 Bundler.require(:default, ENV['KARAFKA_ENV'])
+
+Karafka::Loader.new.load(Karafka::App.root)
 ```
 
 with
@@ -894,6 +963,7 @@ After that make sure that whole your application is loaded before setting up and
 ### Libraries and components
 
 * [Karafka framework](https://github.com/karafka/karafka)
+* [Capistrano Karafka](https://github.com/karafka/capistrano-karafka)
 * [Waterdrop](https://github.com/karafka/waterdrop)
 * [Worker Glass](https://github.com/karafka/worker-glass)
 * [Envlogic](https://github.com/karafka/envlogic)

data/karafka.gemspec

@@ -4,28 +4,29 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'karafka/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = 'karafka'
-  spec.version       = ::Karafka::VERSION
-  spec.platform      = Gem::Platform::RUBY
-  spec.authors       = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
-  spec.email         = %w( maciej@mensfeld.pl pavlo.vavruk@gmail.com adam99g@gmail.com )
-  spec.homepage      = 'https://github.com/karafka/karafka'
-  spec.summary       = %q{ Ruby based framework for working with Apache Kafka }
-  spec.description   = %q{ Framework used to simplify Apache Kafka based Ruby applications development }
-  spec.license       = 'MIT'
+  spec.name        = 'karafka'
+  spec.version     = ::Karafka::VERSION
+  spec.platform    = Gem::Platform::RUBY
+  spec.authors     = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
+  spec.email       = %w( maciej@coditsu.io pavlo.vavruk@gmail.com adam99g@gmail.com )
+  spec.homepage    = 'https://github.com/karafka/karafka'
+  spec.summary     = %q{ Ruby based framework for working with Apache Kafka }
+  spec.description = %q{ Framework used to simplify Apache Kafka based Ruby applications development }
+  spec.license     = 'MIT'
 
   spec.add_development_dependency 'bundler', '~> 1.2'
 
-  spec.add_dependency 'ruby-kafka', '= 0.3.15'
-  spec.add_dependency 'sidekiq', '~> 4.2'
+  spec.add_dependency 'ruby-kafka', '= 0.3.17'
+  spec.add_dependency 'sidekiq', '>= 4.2'
   spec.add_dependency 'worker-glass', '~> 0.2'
   spec.add_dependency 'celluloid', '~> 0.17'
   spec.add_dependency 'envlogic', '~> 1.0'
-  spec.add_dependency 'waterdrop', '~> 0.3.2.1'
+  spec.add_dependency 'waterdrop', '~> 0.3.2.4'
   spec.add_dependency 'rake', '~> 11.3'
   spec.add_dependency 'thor', '~> 0.19'
   spec.add_dependency 'activesupport', '~> 5.0'
-  spec.add_dependency 'dry-configurable', '~> 0.5'
+  spec.add_dependency 'dry-validation', '~> 0.10.6'
+  spec.add_dependency 'dry-configurable', '~> 0.7'
   spec.required_ruby_version = '>= 2.3.0'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }

data/lib/karafka.rb

@@ -16,6 +16,7 @@
   thor
   fileutils
   dry-configurable
+  dry-validation
   active_support/callbacks
   active_support/core_ext/class/subclasses
   active_support/core_ext/hash/indifferent_access
@@ -48,7 +49,7 @@ module Karafka
 
     # @return [String] Karafka app root path (user application path)
     def root
-      Pathname.new(File.dirname(ENV['BUNDLE_GEMFILE']))
+      Pathname.new(ENV['KARAFKA_ROOT_DIR'] || File.dirname(ENV['BUNDLE_GEMFILE']))
     end
 
     # @return [String] path to Karafka gem root core

data/lib/karafka/app.rb

@@ -12,7 +12,10 @@ module Karafka
       # Sets up all the internal components and bootstrap whole app
       # We need to know details about routes in order to setup components,
       # that's why we don't setup them after std setup is done
+      # @raise [Karafka::Errors::InvalidConfiguration] raised when configuration
+      #   doesn't match with ConfigurationSchema
       def boot!
+        Setup::Config.validate!
         Setup::Config.setup_components
       end
 

data/lib/karafka/base_responder.rb

@@ -71,6 +71,16 @@ module Karafka
         topic_obj = Responders::Topic.new(topic_name, options)
         self.topics[topic_obj.name] = topic_obj
       end
+
+      # A simple alias for easier standalone responder usage.
+      # Instead of building it with new.call it allows (in case of usin JSON parser)
+      # to just run it directly from the class level
+      # @param data Anything that we want to respond with
+      # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+      #   UsersCreatedResponder.call(@created_user)
+      def call(*data)
+        new.call(*data)
+      end
     end
 
     # Creates a responder object
@@ -87,6 +97,10 @@ module Karafka
     # @note We know that validators should be executed also before sending data to topics, however
     #   the implementation gets way more complicated then, that's why we check after everything
     #   was sent using responder
+    # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
+    #   UsersCreatedResponder.new.call(@created_user)
+    # @example Send user data with a responder using non default Parser
+    #   UsersCreatedResponder.new(MyParser).call(@created_user)
     def call(*data)
       respond(*data)
       validate!
@@ -133,8 +147,16 @@ module Karafka
     #   what we send is legit and it will go to a proper topics
     def deliver!
       messages_buffer.each do |topic, data_elements|
+        # We map this topic name, so it will match namespaced/etc topic in Kafka
+        # @note By default will not change topic (if default mapper used)
+        mapped_topic = Karafka::App.config.topic_mapper.outgoing(topic)
+
         data_elements.each do |(data, options)|
-          ::WaterDrop::Message.new(topic, data, options).send!
+          ::WaterDrop::Message.new(
+            mapped_topic,
+            data,
+            options
+          ).send!
         end
       end
     end

data/lib/karafka/connection/consumer.rb

@@ -10,10 +10,14 @@ module Karafka
       #   If we would't catch it, it would propagate up until killing the Celluloid actor
       # @param message [Kafka::FetchedMessage] message that was fetched by kafka
       def consume(message)
-        controller = Karafka::Routing::Router.new(message.topic).build
+        # We map from incoming topic name, as it might be namespaced, etc.
+        # @see topic_mapper internal docs
+        mapped_topic = Karafka::App.config.topic_mapper.incoming(message.topic)
+
+        controller = Karafka::Routing::Router.new(mapped_topic).build
         # We wrap it around with our internal message format, so we don't pass around
         # a raw Kafka message
-        controller.params = Message.new(message.topic, message.value)
+        controller.params = Message.new(mapped_topic, message.value)
 
         Karafka.monitor.notice(self.class, controller.to_h)
 

data/lib/karafka/connection/topic_consumer.rb

@@ -3,6 +3,10 @@ module Karafka
     # Class used as a wrapper around Ruby-Kafka to simplify additional
     # features that we provide/might provide in future
     class TopicConsumer
+      # How long should we wait before trying to reconnect to Kafka cluster
+      # that went down (in seconds)
+      RECONNECT_TIMEOUT = 5
+
       # Creates a queue consumer that will pull the data from Kafka
       # @param [Karafka::Routing::Route] route details that will be used to build up a
       #   queue consumer instance
@@ -63,6 +67,13 @@ module Karafka
             start_from_beginning: @route.start_from_beginning
           )
         end
+      rescue Kafka::ConnectionError
+        # If we would not wait it would totally spam log file with failed
+        # attempts if Kafka is down
+        sleep(RECONNECT_TIMEOUT)
+        # We don't log and just reraise - this will be logged
+        # down the road
+        raise
       end
 
       # @return [Kafka] returns a Kafka

data/lib/karafka/errors.rb

@@ -10,9 +10,15 @@ module Karafka
     class ParserError < BaseError; end
 
     # Raised when router receives topic name which does not correspond with any routes
-    #   This should never happen because we listed only to topics defined in routes
-    #   but theory is not always right. If you encounter this error - please contact
-    #   Karafka maintainers
+    # This can only happen in a case when:
+    #   - you've received a message and it was scheduled to Sidekiq background worker
+    #   - you've changed the routing, so router can no longer associate your topic to
+    #     any controller
+    #   - or in a case when you do a lot of metaprogramming and you change routing/etc on runtime
+    #
+    # In case this happens, you will have to create a temporary route that will allow
+    # you to "eat" everything from the Sidekiq queue.
+    # @see https://github.com/karafka/karafka/issues/135
     class NonMatchingRouteError < BaseError; end
 
     # Raised when we have few controllers(inherited from Karafka::BaseController)
@@ -46,5 +52,8 @@ module Karafka
 
     # Raised when we didn't use a topic that was defined as non-optional (required)
     class UnusedResponderRequiredTopic < BaseError; end
+
+    # Raised when configuration doesn't match with validation schema
+    class InvalidConfiguration < BaseError; end
   end
 end

data/lib/karafka/process.rb

@@ -6,7 +6,7 @@ module Karafka
 
     # Signal types that we handle
     HANDLED_SIGNALS = %i(
-      SIGINT SIGQUIT
+      SIGINT SIGQUIT SIGTERM
     ).freeze
 
     HANDLED_SIGNALS.each do |signal|

data/lib/karafka/routing/mapper.rb

@@ -0,0 +1,53 @@
+module Karafka
+  module Routing
+    # Default routes mapper that does not remap things
+    # Mapper can be used for Kafka providers that require namespaced topic names. Instead of being
+    # provider dependent, we can then define mapper and use internally "pure" topic names in
+    # routes and responders
+    #
+    # @example Mapper for mapping prefixed topics
+    #   module MyMapper
+    #     PREFIX = "my_user_name."
+    #
+    #     def incoming(topic)
+    #       topic.to_s.gsub(PREFIX, '')
+    #     end
+    #
+    #     def outgoing(topic)
+    #       "#{PREFIX}#{topic}"
+    #     end
+    #   end
+    #
+    # @example Mapper for replacing "." with "_" in topic names
+    #   module MyMapper
+    #     PREFIX = "my_user_name."
+    #
+    #     def incoming(topic)
+    #       topic.to_s.gsub('.', '_')
+    #     end
+    #
+    #     def outgoing(topic)
+    #       topic.to_s.gsub('_', '.')
+    #     end
+    #   end
+    module Mapper
+      class << self
+        # @param topic [String, Symbol] topic
+        # @return [String, Symbol] same topic as on input
+        # @example
+        #   incoming('topic_name') #=> 'topic_name'
+        def incoming(topic)
+          topic
+        end
+
+        # @param topic [String, Symbol] topic
+        # @return [String, Symbol] same topic as on input
+        # @example
+        #   outgoing('topic_name') #=> 'topic_name'
+        def outgoing(topic)
+          topic
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/route.rb

@@ -56,7 +56,7 @@ module Karafka
       #   background job
       # @note If not provided - will be built based on the provided controller
       def worker
-        @worker ||= Karafka::Workers::Builder.new(controller).build
+        @worker ||= inline_mode ? nil : Karafka::Workers::Builder.new(controller).build
       end
 
       # @return [Class, nil] Class (not an instance) of a responder that should respond from

data/lib/karafka/server.rb

@@ -11,6 +11,7 @@ module Karafka
         @consumers = Concurrent::Array.new
         bind_on_sigint
         bind_on_sigquit
+        bind_on_sigterm
         start_supervised
       end
 
@@ -39,6 +40,15 @@ module Karafka
         end
       end
 
+      # What should happen when we decide to quit with sigterm
+      def bind_on_sigterm
+        process.on_sigterm do
+          Karafka::App.stop!
+          consumers.map(&:stop)
+          exit
+        end
+      end
+
       # Starts Karafka with a supervision
       # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
       # finish loop (and it won't happen until we explicitily want to stop)

data/lib/karafka/setup/config.rb

@@ -29,6 +29,12 @@ module Karafka
       setting :batch_mode, false
       #  whether to consume messages starting at the beginning or to just consume new messages
       setting :start_from_beginning, true
+      # Mapper used to remap names of topics, so we can have a clean internal topic namings despite
+      # using any Kafka provider that uses namespacing, etc
+      # It needs to implement two methods:
+      #   - #incoming - for remapping from the incoming message to our internal format
+      #   - #outgoing - for remapping from internal topic name into outgoing message
+      setting :topic_mapper, Routing::Mapper
 
       # Connection pool options are used for producer (Waterdrop)
       # They are configured automatically based on Sidekiq concurrency and number of routes
@@ -95,6 +101,18 @@ module Karafka
             klass.new(config).setup
           end
         end
+
+        # Validate config based on ConfigurationSchema
+        # @return [Boolean] true if configuration is valid
+        # @raise [Karafka::Errors::InvalidConfiguration] raised when configuration
+        #   doesn't match with ConfigurationSchema
+        def validate!
+          validation_result = Karafka::Setup::ConfigSchema.call(config.to_h)
+
+          return true if validation_result.success?
+
+          raise Errors::InvalidConfiguration, validation_result.errors
+        end
       end
     end
   end

data/lib/karafka/setup/config_schema.rb

@@ -0,0 +1,44 @@
+module Karafka
+  module Setup
+    # Schema with validation rules for all configuration
+    ConfigSchema = Dry::Validation.Schema do
+      required(:name).filled(:str?)
+      required(:topic_mapper).filled
+      optional(:inline_mode).filled(:bool?)
+
+      required(:redis).maybe do
+        schema do
+          required(:url).filled(:str?)
+        end
+      end
+
+      # If inline_mode is true, redis should be filled
+      rule(redis_presence: [:redis, :inline_mode]) do |redis, inline_mode|
+        inline_mode.false?.then(redis.filled?)
+      end
+
+      optional(:batch_mode).filled(:bool?)
+      optional(:start_from_beginning).filled(:bool?)
+
+      optional(:connection_pool).schema do
+        required(:size).filled
+        optional(:timeout).filled(:int?)
+      end
+
+      required(:kafka).schema do
+        required(:hosts).filled(:array?)
+
+        required(:session_timeout).filled(:int?)
+        required(:offset_commit_interval).filled(:int?)
+        required(:offset_commit_threshold).filled(:int?)
+        required(:heartbeat_interval).filled(:int?)
+
+        optional(:ssl).schema do
+          required(:ca_cert).maybe(:str?)
+          required(:client_cert).maybe(:str?)
+          required(:client_cert_key).maybe(:str?)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/version.rb

@@ -2,5 +2,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '0.5.0.2'
+  VERSION = '0.5.0.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 0.5.0.2
+  version: 0.5.0.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-09 00:00:00.000000000 Z
+date: 2017-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -32,26 +32,26 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.15
+        version: 0.3.17
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.15
+        version: 0.3.17
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.2'
 - !ruby/object:Gem::Dependency
@@ -102,14 +102,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.2.1
+        version: 0.3.2.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.2.1
+        version: 0.3.2.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -152,23 +152,37 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: dry-validation
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.6
 - !ruby/object:Gem::Dependency
   name: dry-configurable
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.7'
 description: " Framework used to simplify Apache Kafka based Ruby applications development "
 email:
-- maciej@mensfeld.pl
+- maciej@coditsu.io
 - pavlo.vavruk@gmail.com
 - adam99g@gmail.com
 executables:
@@ -194,8 +208,6 @@ files:
 - lib/karafka/base_controller.rb
 - lib/karafka/base_responder.rb
 - lib/karafka/base_worker.rb
-- lib/karafka/capistrano.rb
-- lib/karafka/capistrano/karafka.cap
 - lib/karafka/cli.rb
 - lib/karafka/cli/base.rb
 - lib/karafka/cli/console.rb
@@ -225,10 +237,12 @@ files:
 - lib/karafka/responders/topic.rb
 - lib/karafka/responders/usage_validator.rb
 - lib/karafka/routing/builder.rb
+- lib/karafka/routing/mapper.rb
 - lib/karafka/routing/route.rb
 - lib/karafka/routing/router.rb
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb
+- lib/karafka/setup/config_schema.rb
 - lib/karafka/setup/configurators/base.rb
 - lib/karafka/setup/configurators/celluloid.rb
 - lib/karafka/setup/configurators/sidekiq.rb
@@ -264,7 +278,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka

data/lib/karafka/capistrano.rb

@@ -1,2 +0,0 @@
-# Load Capistrano tasks only if Capistrano is loaded
-load File.expand_path('../capistrano/karafka.cap', __FILE__) if defined?(Capistrano)

data/lib/karafka/capistrano/karafka.cap

@@ -1,85 +0,0 @@
-# @note Inspired by Puma capistrano handlers
-# @see https://github.com/seuros/capistrano-puma/blob/master/lib/capistrano/tasks/puma.rake
-namespace :load do
-  task :defaults do
-    set :karafka_role, :app
-    set :karafka_default_hooks, -> { true }
-    set :karafka_env, -> { fetch(:karafka_env, fetch(:environment)) }
-    set :karafka_pid, -> { File.join(shared_path, 'tmp', 'pids', 'karafka.pid') }
-  end
-end
-
-namespace :deploy do
-  before :starting, :check_karafka_hooks do
-    invoke 'karafka:add_default_hooks' if fetch(:karafka_default_hooks)
-  end
-end
-
-namespace :karafka do
-  desc 'Stop Karafka'
-  task :stop do
-    on roles(fetch(:karafka_role)) do |host|
-      within shared_path do
-        # If there's no pidfile it means that Karafka is not running
-        next unless test "cat #{fetch(:karafka_pid)}"
-
-        # Send a kill signal to a given process
-        execute "kill -INT `cat #{fetch(:karafka_pid)}`"
-
-        # And wait until it finishes. We wait because we don't want to start next process until
-        # the previous one is stopped. That way we won't have problems with Kafka registering and
-        # deregistering processes from topics (although nothing bad would happen. It would just
-        # take more time to rebalance)
-        while true
-          break unless test "cat #{fetch(:karafka_pid)}"
-          info 'Waiting for Karafka to stop'
-          sleep 5
-        end
-      end
-    end
-  end
-
-  desc 'Start Karafka'
-  task :start do
-    on roles(fetch(:karafka_role)) do |host|
-      within current_path do
-        # We use all 3 because when combined with Sinatra/Rails it will use their parts as well
-        # so we want to set proper env for any of them
-        with(
-          KARAFKA_ENV: fetch(:karafka_env),
-          RAILS_ENV: fetch(:rails_env),
-          RACK_ENV: fetch(:rack_env)
-        )do
-          execute :bundle, "exec karafka server -d -p #{fetch(:karafka_pid)}"
-        end
-      end
-    end
-  end
-
-  desc 'Restart Karafka'
-  task :restart do
-    invoke 'karafka:stop'
-    invoke 'karafka:start'
-  end
-
-  desc 'Status Karafka'
-  task :status do
-    on roles(fetch(:karafka_role)) do |host|
-      if test "cat #{fetch(:karafka_pid)}"
-        pid = capture "cat #{fetch(:karafka_pid)}"
-
-        if test "ps -p #{pid} > /dev/null"
-          info "Karafka is started: #{pid}"
-        else
-          error "Karafka is not started but pidfile exists"
-        end
-      else
-        info "Karafka is not started"
-      end
-    end
-  end
-
-  task :add_default_hooks do
-    after 'deploy:finished', 'karafka:restart'
-  end
-end