karafka 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49921e358054ab6470899c0e122cc9c93f5398ea
-  data.tar.gz: 183d5ff52e6f4ce09110287147d735929ff2a6e7
+  metadata.gz: c7d3b30943ab27b3ae2d8609e645590ca918ff2b
+  data.tar.gz: 0b6b31b10d070c07d45ab6bb8bc177e94d4d9c3c
 SHA512:
-  metadata.gz: cd8ad63256d3133dbc1c4a4676b54c64815353047a8cb42c87e6ef343ff6af12099ff07d03d01b4a3b61fc648d79dc6b13c1f969a48f3ab04f1dc3a91799e99b
-  data.tar.gz: 345addc070e2b13fc7d11f07faefccbcfbffeedad3ce106d98a335c0e54f5c29731fc9288c46b114aee4df62f7578c2e120e8b61b7a378c517881221b2254e4e
+  metadata.gz: b8a3f16977e6af10f6cda783c326be7bb8540b483cacfe225fc7c422cad091d9014950c6a6de4a9f5cd033b974a41de73bc445e7813063d12013684f056d35df
+  data.tar.gz: 2e425da4e9e420ad092b3b563e5fa796f3f73159bc9c5d6cdf77bbf8463842f0a9379fe156b54328a3f65144f08ec657fc6f65aa01295a3efc40181e4bd0d858

data/.ruby-version

@@ -1 +1 @@
-2.4.1
+2.4.2

data/.travis.yml

@@ -8,7 +8,8 @@ rvm:
   - 2.3.4
   - 2.4.0
   - 2.4.1
-  - jruby-9.1.12.0
+  - 2.4.2
+  - jruby-head
 script: bundle exec rspec spec/
 env:
   global:

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Karafka framework changelog
 
+## 1.0.1
+- #210 - LoadError: cannot load such file -- [...]/karafka.rb
+- Ruby 2.4.2 as a default (+travis integration)
+- JRuby upgrade
+- Expanded persistence layer (moved to a namespace for easier future development)
+- #213 - Misleading error when non-existing dependency is required
+- #212 - Make params react to #topic, #partition, #offset
+- #215 - Consumer group route dynamic options are ignored
+- #217 - check RUBY_ENGINE constant if RUBY_VERSION is missing (#217)
+- #218 - add configuration setting to control Celluloid's shutdown timeout
+- Renamed Karafka::Routing::Mapper to Karafka::Routing::TopicMapper to match naming conventions
+- #219 - Allow explicit consumer group names, without prefixes
+- Fix to early removed pid upon shutdown of demonized process
+- max_wait_time updated to match https://github.com/zendesk/ruby-kafka/issues/433
+- #230 - Better uri validation for seed brokers (incompatibility as the kafka:// or kafka+ssl:// is required)
+- Small internal docs fixes
+- Dry::Validation::MissingMessageError: message for broker_schema? was not found
+- #238 - warning: already initialized constant Karafka::Schemas::URI_SCHEMES
+
 ## 1.0.0
 
 ### Closed issues:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (1.0.0)
+    karafka (1.0.1)
       activesupport (>= 5.0)
       celluloid
       dry-configurable (~> 0.7)
@@ -12,12 +12,12 @@ PATH
       require_all (>= 1.4)
       ruby-kafka (>= 0.4)
       thor (~> 0.19)
-      waterdrop (>= 0.4)
+      waterdrop (~> 0.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.3)
+    activesupport (5.1.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
@@ -48,54 +48,55 @@ GEM
     dry-container (0.6.0)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.3.3)
+    dry-core (0.3.4)
       concurrent-ruby (~> 1.0)
     dry-equalizer (0.2.0)
-    dry-logic (0.4.1)
+    dry-logic (0.4.2)
       dry-container (~> 0.2, >= 0.2.6)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
-    dry-types (0.11.1)
+    dry-types (0.12.1)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1)
       dry-container (~> 0.3)
       dry-core (~> 0.2, >= 0.2.1)
       dry-equalizer (~> 0.2)
-      dry-logic (~> 0.4, >= 0.4.0)
+      dry-logic (~> 0.4, >= 0.4.2)
       inflecto (~> 0.0.0, >= 0.0.2)
-    dry-validation (0.11.0)
+    dry-validation (0.11.1)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
       dry-core (~> 0.2, >= 0.2.1)
       dry-equalizer (~> 0.2)
       dry-logic (~> 0.4, >= 0.4.0)
-      dry-types (~> 0.11.0)
+      dry-types (~> 0.12.0)
     envlogic (1.0.4)
       activesupport
     hitimes (1.2.6)
-    i18n (0.8.6)
+    i18n (0.9.0)
+      concurrent-ruby (~> 1.0)
     inflecto (0.0.2)
     json (2.1.0)
     minitest (5.10.3)
     multi_json (1.12.2)
     null-logger (0.1.4)
-    rake (12.0.0)
+    rake (12.2.1)
     require_all (1.4.0)
-    rspec (3.6.0)
-      rspec-core (~> 3.6.0)
-      rspec-expectations (~> 3.6.0)
-      rspec-mocks (~> 3.6.0)
-    rspec-core (3.6.0)
-      rspec-support (~> 3.6.0)
-    rspec-expectations (3.6.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.0)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.6.0)
-    rspec-mocks (3.6.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.6.0)
-    rspec-support (3.6.0)
-    ruby-kafka (0.4.1)
-    simplecov (0.15.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
+    ruby-kafka (0.4.3)
+    simplecov (0.15.1)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
@@ -105,7 +106,7 @@ GEM
     timecop (0.9.1)
     timers (4.1.2)
       hitimes
-    tzinfo (1.2.3)
+    tzinfo (1.2.4)
       thread_safe (~> 0.1)
     waterdrop (0.4.0)
       bundler
@@ -125,4 +126,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   1.14.6
+   1.15.4

data/README.md

@@ -1,6 +1,6 @@
 ![karafka logo](https://raw.githubusercontent.com/karafka/misc/master/logo/karafka_logotype_transparent2.png)
 
-[![Build Status](https://travis-ci.org/karafka/karafka.png)](https://travis-ci.org/karafka/karafka)
+[![Build Status](https://travis-ci.org/karafka/karafka.svg?branch=master)](https://travis-ci.org/karafka/karafka)
 [![Backers on Open Collective](https://opencollective.com/karafka/backers/badge.svg)](#backers) [![Sponsors on Open Collective](https://opencollective.com/karafka/sponsors/badge.svg)](#sponsors) [![Join the chat at https://gitter.im/karafka/karafka](https://badges.gitter.im/karafka/karafka.svg)](https://gitter.im/karafka/karafka?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 Framework used to simplify Apache Kafka based Ruby applications development.
@@ -11,7 +11,7 @@ Karafka not only handles incoming messages but also provides tools for building
 
 ## How does it work
 
-Karafka provides a higher-level abstraction that allows you to focus on your business logic development, instead of focusing on  implementing lower level abstration layers. It provides developers with a set of tools that are dedicated for building multi-topic applications similarly to how Rails applications are being built.
+Karafka provides a higher-level abstraction that allows you to focus on your business logic development, instead of focusing on implementing lower level abstraction layers. It provides developers with a set of tools that are dedicated for building multi-topic applications similarly to how Rails applications are being built.
 
 Karafka based applications can be easily deployed to any type of infrastructure, including those based on:
 
@@ -73,7 +73,7 @@ Commit, do not mess with Rakefile, version, or history. (if you want to have you
 
 [![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/karafka/repositories/karafka)
 
-Each pull request must pass our quality requirements. To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combinse multiple linters and code analyzers for both code and documentation.
+Each pull request must pass our quality requirements. To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation.
 
 Unfortunately, it does not yet support independent forks, however you should be fine by looking at what we require.
 

data/bin/karafka

@@ -1,7 +1,19 @@
 #!/usr/bin/env ruby
 
 require 'karafka'
-require Karafka.boot_file.to_s
+
+# If there is a boot file, we need to require it as we expect it to contain
+# Karafka app setup, routes, etc
+if File.exist?(Karafka.boot_file)
+  require Karafka.boot_file.to_s
+else
+  # However when it is unavailable, we still want to be able to run help command
+  # and install command as they don't require configured app itself to run
+  raise(
+    Karafka::Errors::MissingBootFile,
+    Karafka.boot_file
+  ) unless %w[-h install].include?(ARGV[0])
+end
 
 Karafka::Cli.prepare
 Karafka::Cli.start

data/config/errors.yml

@@ -0,0 +1,6 @@
+en:
+  errors:
+    broker_schema?: >
+      has an invalid format.
+      Expected schema, host and port number.
+      Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092

data/karafka.gemspec

@@ -16,17 +16,17 @@ Gem::Specification.new do |spec|
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.license     = 'MIT'
 
-  spec.add_dependency 'ruby-kafka', '>= 0.4'
-  spec.add_dependency 'celluloid'
-  spec.add_dependency 'envlogic', '~> 1.0'
-  spec.add_dependency 'waterdrop', '>= 0.4'
-  spec.add_dependency 'rake', '>= 11.3'
-  spec.add_dependency 'thor', '~> 0.19'
   spec.add_dependency 'activesupport', '>= 5.0'
-  spec.add_dependency 'dry-validation', '~> 0.11'
+  spec.add_dependency 'celluloid'
   spec.add_dependency 'dry-configurable', '~> 0.7'
+  spec.add_dependency 'dry-validation', '~> 0.11'
+  spec.add_dependency 'envlogic', '~> 1.0'
   spec.add_dependency 'multi_json', '>= 1.12'
+  spec.add_dependency 'rake', '>= 11.3'
   spec.add_dependency 'require_all', '>= 1.4'
+  spec.add_dependency 'ruby-kafka', '>= 0.4'
+  spec.add_dependency 'thor', '~> 0.19'
+  spec.add_dependency 'waterdrop', '~> 0.4'
 
   spec.required_ruby_version = '>= 2.3.0'
 

data/lib/karafka/base_controller.rb

@@ -53,7 +53,7 @@ module Karafka
     # @see http://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-get_callbacks
     define_callbacks :after_received
 
-    attr_accessor :params_batch
+    attr_reader :params_batch
 
     class << self
       attr_reader :topic
@@ -65,7 +65,6 @@ module Karafka
       def topic=(topic)
         @topic = topic
         Controllers::Includer.call(self)
-        @topic
       end
 
       # Creates a callback that will be executed after receiving message but before executing the

data/lib/karafka/cli/base.rb

@@ -18,6 +18,8 @@ module Karafka
     #     end
     #   end
     class Base
+      include Thor::Shell
+
       # We can use it to call other cli methods via this object
       attr_reader :cli
 

data/lib/karafka/cli/flow.rb

@@ -15,7 +15,7 @@ module Karafka
           if any_topics
             puts "#{topic.name} =>"
 
-            topic.responder.topics.each do |_name, responder_topic|
+            topic.responder.topics.each_value do |responder_topic|
               features = []
               features << (responder_topic.required? ? 'always' : 'conditionally')
               features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')

data/lib/karafka/cli/server.rb

@@ -30,13 +30,13 @@ module Karafka
           Celluloid.boot
         end
 
-        # Remove pidfile on shutdown
-        ObjectSpace.define_finalizer(String.new, proc { send(:clean) })
-
         # We assign active topics on a server level, as only server is expected to listen on
         # part of the topics
         Karafka::Server.consumer_groups = cli.options[:consumer_groups]
 
+        # Remove pidfile on shutdown, just before the server instance is going to be GCed
+        ObjectSpace.define_finalizer(self, proc { send(:clean) })
+
         # After we fork, we can boot celluloid again
         Karafka::Server.run
       end
@@ -62,7 +62,7 @@ module Karafka
 
       # Removes a pidfile (if exist)
       def clean
-        FileUtils.rm_f(cli.options[:pid])
+        FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
       end
     end
   end

data/lib/karafka/connection/config_adapter.rb

@@ -41,25 +41,24 @@ module Karafka
         # @return [Hash] hash with all the settings required by Kafka#consumer method
         def consumer(consumer_group)
           settings = { group_id: consumer_group.id }
-          settings = fetch_for(:consumer, settings)
+          settings = fetch_for(:consumer, consumer_group, settings)
           sanitize(settings)
         end
 
         # Builds all the configuration settings for kafka consumer consume_each_batch and
         #   consume_each_message methods
-        # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
         # @return [Hash] hash with all the settings required by
         #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
-        def consuming(_consumer_group)
-          sanitize(fetch_for(:consuming))
+        def consuming(consumer_group)
+          sanitize(fetch_for(:consuming, consumer_group))
         end
 
         # Builds all the configuration settings for kafka consumer#subscribe method
         # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
         # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
         def subscription(topic)
-          settings = { start_from_beginning: topic.start_from_beginning }
-          settings = fetch_for(:subscription, settings)
+          settings = fetch_for(:subscription, topic)
           [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
         end
 
@@ -74,13 +73,19 @@ module Karafka
 
         # Fetches proper settings for a given map namespace
         # @param namespace_key [Symbol] namespace from attributes map config adapter hash
+        # @param route_layer [Object] route topic or consumer group
         # @param preexisting_settings [Hash] hash with some preexisting settings that might have
         #   been loaded in a different way
-        def fetch_for(namespace_key, preexisting_settings = {})
-          kafka_configs.each do |setting_name, setting_value|
+        def fetch_for(namespace_key, route_layer, preexisting_settings = {})
+          kafka_configs.each_key do |setting_name|
+            # Ignore settings that are not related to our namespace
             next unless AttributesMap.config_adapter[namespace_key].include?(setting_name)
+            # Ignore settings that are already initialized
+            # In case they are in preexisting settings fetched differently
             next if preexisting_settings.keys.include?(setting_name)
-            preexisting_settings[setting_name] = setting_value
+            # Fetch all the settings from a given layer object. Objects can handle the fallback
+            # to the kafka settings, so
+            preexisting_settings[setting_name] = route_layer.send(setting_name)
           end
 
           preexisting_settings

data/lib/karafka/connection/messages_processor.rb

@@ -20,15 +20,17 @@ module Karafka
           # @see topic_mapper internal docs
           mapped_topic_name = Karafka::App.config.topic_mapper.incoming(kafka_messages[0].topic)
           topic = Routing::Router.find("#{group_id}_#{mapped_topic_name}")
-
-          # Depending on a case (persisted or not) we might use new controller instance per each
-          # batch, or use the same instance for all of them (for implementing buffering, etc)
-          controller = Persistence.fetch(topic, kafka_messages[0].partition, :controller) do
+          controller = Persistence::Controller.fetch(topic, kafka_messages[0].partition) do
             topic.controller.new
           end
 
-          handler = topic.batch_processing ? :process_batch : :process_each
-          send(handler, controller, kafka_messages)
+          # Depending on a case (persisted or not) we might use new controller instance per each
+          # batch, or use the same instance for all of them (for implementing buffering, etc)
+          send(
+            topic.batch_processing ? :process_batch : :process_each,
+            controller,
+            kafka_messages
+          )
         end
 
         private

data/lib/karafka/errors.rb

@@ -29,5 +29,8 @@ module Karafka
 
     # Raised when configuration doesn't match with validation schema
     InvalidConfiguration = Class.new(BaseError)
+
+    # Raised when we try to use Karafka CLI commands (except install) without a bootfile
+    MissingBootFile = Class.new(BaseError)
   end
 end

data/lib/karafka/params/params.rb

@@ -16,6 +16,18 @@ module Karafka
         key
       ].freeze
 
+      # Params attributes that should be available via a method call invocation for Kafka
+      # client compatibility.
+      # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
+      # uses those fields via method calls, so in order to be able to pass there our params
+      # objects, have to have same api.
+      PARAMS_METHOD_ATTRIBUTES = %i[
+        topic
+        partition
+        offset
+        key
+      ].freeze
+
       class << self
         # We allow building instances only via the #build method
 
@@ -51,6 +63,18 @@ module Karafka
             end
           end
         end
+
+        # Defines a method call accessor to a particular hash field.
+        # @note Won't work for complex key names that contain spaces, etc
+        # @param key [Symbol] name of a field that we want to retrieve with a method call
+        # @example
+        #   key_attr_reader :example
+        #   params.example #=> 'my example value'
+        def key_attr_reader(key)
+          define_method key do
+            self[key]
+          end
+        end
       end
 
       # @return [Karafka::Params::Params] this will trigger parser execution. If we decide to
@@ -63,6 +87,7 @@ module Karafka
         merge!(parse(delete(:value)))
       end
 
+      PARAMS_METHOD_ATTRIBUTES.each(&method(:key_attr_reader))
 
       private
 

data/lib/karafka/patches/dry_configurable.rb

@@ -13,9 +13,7 @@ module Karafka
       def initialize(*args)
         super
 
-        @config.each do |key, _value|
-          rebuild(key)
-        end
+        @config.each_key(&method(:rebuild))
       end
 
       private

data/lib/karafka/persistence/controller.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module used to provide a persistent cache layer for Karafka components that need to be
+  # shared inside of a same thread
+  module Persistence
+    # Module used to provide a persistent cache across batch requests for a given
+    # topic and partition to store some additional details when the persistent mode
+    # for a given topic is turned on
+    class Controller
+      # Used to build (if block given) and/or fetch a current controller instance that will be used
+      #   to process messages from a given topic and partition
+      # @return [Karafka::BaseController] base controller descendant
+      # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
+      # @param partition [Integer] number of partition for which we want to cache
+      def self.fetch(topic, partition)
+        return yield unless topic.persistent
+        Thread.current[topic.id] ||= {}
+        Thread.current[topic.id][partition] ||= yield
+      end
+    end
+  end
+end

data/lib/karafka/routing/builder.rb

@@ -28,7 +28,7 @@ module Karafka
           hashed_group = consumer_group.to_h
           validation_result = Karafka::Schemas::ConsumerGroup.call(hashed_group)
           return if validation_result.success?
-          raise Errors::InvalidConfiguration, [validation_result.errors, hashed_group]
+          raise Errors::InvalidConfiguration, validation_result.errors
         end
       end
 

data/lib/karafka/routing/consumer_group.rb

@@ -18,7 +18,7 @@ module Karafka
       #   kafka and don't understand the concept of consumer groups.
       def initialize(name)
         @name = name
-        @id = "#{Karafka::App.config.client_id.to_s.underscore}_#{@name}"
+        @id = Karafka::App.config.consumer_mapper.call(name)
         @topics = []
       end
 

data/lib/karafka/routing/consumer_mapper.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Default consumer mapper that builds consumer ids based on app id and consumer group name
+    # Different mapper can be used in case of preexisting consumer names or for applying
+    # other naming conventions not compatible wiih Karafkas client_id + consumer name concept
+    #
+    # @example Mapper for using consumer groups without a client_id prefix
+    #   module MyMapper
+    #     def self.call(raw_consumer_group_name)
+    #       raw_consumer_group_name
+    #     end
+    #   end
+    #
+    # @example Mapper for replacing "_" with "." in topic names
+    #   module MyMapper
+    #     def self.call(raw_consumer_group_name)
+    #       [
+    #         Karafka::App.config.client_id.to_s.underscope,
+    #         raw_consumer_group_name
+    #       ].join('_').gsub('_', '.')
+    #     end
+    #   end
+    module ConsumerMapper
+      # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
+      # @return [String] remapped final consumer group name
+      def self.call(raw_consumer_group_name)
+        "#{Karafka::App.config.client_id.to_s.underscore}_#{raw_consumer_group_name}"
+      end
+    end
+  end
+end

data/lib/karafka/routing/{mapper.rb → topic_mapper.rb}

@@ -2,7 +2,7 @@
 
 module Karafka
   module Routing
-    # Default routes mapper that does not remap things
+    # Default topic mapper that does not remap things
     # Mapper can be used for Kafka providers that require namespaced topic names. Instead of being
     # provider dependent, we can then define mapper and use internally "pure" topic names in
     # routes and responders
@@ -32,7 +32,7 @@ module Karafka
     #       topic.to_s.gsub('_', '.')
     #     end
     #   end
-    module Mapper
+    module TopicMapper
       class << self
         # @param topic [String, Symbol] topic
         # @return [String, Symbol] same topic as on input

data/lib/karafka/schemas/config.rb

@@ -13,6 +13,12 @@ module Karafka
     #   so we validate all of that once all the routes are defined and ready
     Config = Dry::Validation.Schema do
       required(:client_id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:consumer_mapper)
+      required(:topic_mapper)
+
+      required(:celluloid).schema do
+        required(:shutdown_timeout).filled(:int?, gteq?: 0)
+      end
 
       optional(:backend).filled
 

data/lib/karafka/schemas/consumer_group.rb

@@ -2,23 +2,31 @@
 
 module Karafka
   module Schemas
-    # Consumer group topic validation rules
-    ConsumerGroupTopic = Dry::Validation.Schema do
-      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:backend).filled(included_in?: %i[inline sidekiq])
-      required(:controller).filled
-      required(:parser).filled
-      required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
-      required(:start_from_beginning).filled(:bool?)
-      required(:batch_processing).filled(:bool?)
-      required(:persistent).filled(:bool?)
-    end
-
     # Schema for single full route (consumer group + topics) validation.
     ConsumerGroup = Dry::Validation.Schema do
+      # Valid uri schemas of Kafka broker url
+      # The ||= is due to the behavior of require_all that resolves dependencies
+      # but someetimes loads things twice
+      URI_SCHEMES ||= %w[kafka kafka+ssl].freeze
+
+      configure do
+        config.messages_file = File.join(
+          Karafka.gem_root, 'config', 'errors.yml'
+        )
+
+        # Uri validator to check if uri is in a Karafka acceptable format
+        # @param uri [String] uri we want to validate
+        # @return [Boolean] true if it is a valid uri, otherwise false
+        def broker_schema?(uri)
+          uri = URI.parse(uri)
+          URI_SCHEMES.include?(uri.scheme) && uri.port
+        rescue URI::InvalidURIError
+          return false
+        end
+      end
+
       required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:seed_brokers).filled(:array?)
+      required(:seed_brokers).filled { each(:broker_schema?) }
       required(:session_timeout).filled(:int?)
       required(:pause_timeout).filled(:int?, gteq?: 0)
       required(:offset_commit_interval).filled(:int?)

data/lib/karafka/schemas/consumer_group_topic.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Schemas
+    # Consumer group topic validation rules
+    ConsumerGroupTopic = Dry::Validation.Schema do
+      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:backend).filled(included_in?: %i[inline sidekiq])
+      required(:controller).filled
+      required(:parser).filled
+      required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
+      required(:start_from_beginning).filled(:bool?)
+      required(:batch_processing).filled(:bool?)
+      required(:persistent).filled(:bool?)
+    end
+  end
+end

data/lib/karafka/setup/config.rb

@@ -24,12 +24,16 @@ module Karafka
       setting :logger, -> { ::Karafka::Logger.instance }
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
       setting :monitor, -> { ::Karafka::Monitor.instance }
+      # Mapper used to remap consumer groups ids, so in case users migrate from other tools
+      # or they need to maintain their own internal consumer group naming conventions, they
+      # can easily do it, replacing the default client_id + consumer name pattern concept
+      setting :consumer_mapper, -> { Routing::ConsumerMapper }
       # Mapper used to remap names of topics, so we can have a clean internal topic namings
       # despite using any Kafka provider that uses namespacing, etc
       # It needs to implement two methods:
       #   - #incoming - for remapping from the incoming message to our internal format
       #   - #outgoing - for remapping from internal topic name into outgoing message
-      setting :topic_mapper, -> { Routing::Mapper }
+      setting :topic_mapper, -> { Routing::TopicMapper }
       # If batch_consuming is true, we will consume kafka messages in batches instead of 1 by 1
       # @note Consuming does not equal processing, see batch_processing description for details
       setting :batch_consuming, true
@@ -43,6 +47,18 @@ module Karafka
       # incoming batch. It's disabled by default, not to create more objects that needed on
       # each batch
       setting :persistent, true
+      # This is configured automatically, don't overwrite it!
+      # Each consumer group requires separate thread, so number of threads should be equal to
+      # number of consumer groups
+      setting :concurrency, -> { ::Karafka::App.consumer_groups.count }
+
+      # option celluloid [Hash] - optional - celluloid configuration options
+      setting :celluloid do
+        # options shutdown_timeout [Integer] How many seconds should we wait for actors (listeners)
+        # before forcefully shutting them
+        setting :shutdown_timeout, 30
+      end
+
       # Connection pool options are used for producer (Waterdrop) - by default it will adapt to
       # number of active actors
       setting :connection_pool do
@@ -86,9 +102,14 @@ module Karafka
         #   returning messages from the server; if `max_wait_time` is reached, this
         #   is ignored.
         setting :min_bytes, 1
-        # option max_wait_time [Integer, Float] the maximum duration of time to wait before
-        #   returning messages from the server, in seconds.
-        setting :max_wait_time, 5
+        # option max_wait_time [Integer, Float] max_wait_time is the maximum number of seconds to
+        #   wait before returning data from a single message fetch. By setting this high you also
+        #   increase the processing throughput – and by setting it low you set a bound on latency.
+        #   This configuration overrides `min_bytes`, so you'll _always_ get data back within the
+        #   time specified. The default value is one second. If you want to have at most five
+        #   seconds of latency, set `max_wait_time` to 5. You should make sure
+        #   max_wait_time * num brokers + heartbeat_interval is less than session_timeout.
+        setting :max_wait_time, 1
         # option reconnect_timeout [Integer] How long should we wait before trying to reconnect to
         # Kafka cluster that went down (in seconds)
         setting :reconnect_timeout, 5
@@ -109,9 +130,9 @@ module Karafka
         setting :ssl_ca_cert, nil
         # option ssl_ca_cert_file_path [String] SSL CA certificate file path
         setting :ssl_ca_cert_file_path, nil
-        # option client_cert [String] SSL client certificate
+        # option ssl_client_cert [String] SSL client certificate
         setting :ssl_client_cert, nil
-        # option client_cert_key [String] SSL client certificate password
+        # option ssl_client_cert_key [String] SSL client certificate password
         setting :ssl_client_cert_key, nil
         # option sasl_gssapi_principal [String] sasl principal
         setting :sasl_gssapi_principal, nil
@@ -125,11 +146,6 @@ module Karafka
         setting :sasl_plain_password, nil
       end
 
-      # This is configured automatically, don't overwrite it!
-      # Each consumer group requires separate thread, so number of threads should be equal to
-      # number of consumer groups
-      setting :concurrency, -> { ::Karafka::App.consumer_groups.count }
-
       class << self
         # Configurating method
         # @yield Runs a block of code providing a config singleton instance to it

data/lib/karafka/setup/configurators/celluloid.rb

@@ -5,16 +5,13 @@ module Karafka
     class Configurators
       # Class responsible for setting up Celluloid settings
       class Celluloid < Base
-        # How many seconds should we wait for actors (listeners) before forcefully shutting them
-        SHUTDOWN_TIME = 30
-
         # Sets up a Karafka logger as celluloid logger
         def setup
           ::Celluloid.logger = ::Karafka.logger
           # This is just a precaution - it should automatically close the current
           # connection and shutdown actor - but in case it didn't (hanged, etc)
           # we will kill it after waiting for some time
-          ::Celluloid.shutdown_timeout = SHUTDOWN_TIME
+          ::Celluloid.shutdown_timeout = config.celluloid.shutdown_timeout
         end
       end
     end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.0.0'
+  VERSION = '1.0.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -10,22 +10,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-06 00:00:00.000000000 Z
+date: 2017-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: celluloid
   requirement: !ruby/object:Gem::Requirement
@@ -41,131 +41,131 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: envlogic
+  name: dry-configurable
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.7'
 - !ruby/object:Gem::Dependency
-  name: waterdrop
+  name: dry-validation
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.11'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: envlogic
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '11.3'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '11.3'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '1.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '1.12'
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '11.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '11.3'
 - !ruby/object:Gem::Dependency
-  name: dry-validation
+  name: require_all
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
-  name: dry-configurable
+  name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
-  name: multi_json
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '0.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '0.19'
 - !ruby/object:Gem::Dependency
-  name: require_all
+  name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '0.4'
 description: Framework used to simplify Apache Kafka based Ruby applications development
 email:
 - maciej@coditsu.io
@@ -192,6 +192,7 @@ files:
 - README.md
 - Rakefile
 - bin/karafka
+- config/errors.yml
 - karafka.gemspec
 - lib/karafka.rb
 - lib/karafka/app.rb
@@ -225,18 +226,20 @@ files:
 - lib/karafka/params/params_batch.rb
 - lib/karafka/parsers/json.rb
 - lib/karafka/patches/dry_configurable.rb
-- lib/karafka/persistence.rb
+- lib/karafka/persistence/controller.rb
 - lib/karafka/process.rb
 - lib/karafka/responders/builder.rb
 - lib/karafka/responders/topic.rb
 - lib/karafka/routing/builder.rb
 - lib/karafka/routing/consumer_group.rb
-- lib/karafka/routing/mapper.rb
+- lib/karafka/routing/consumer_mapper.rb
 - lib/karafka/routing/proxy.rb
 - lib/karafka/routing/router.rb
 - lib/karafka/routing/topic.rb
+- lib/karafka/routing/topic_mapper.rb
 - lib/karafka/schemas/config.rb
 - lib/karafka/schemas/consumer_group.rb
+- lib/karafka/schemas/consumer_group_topic.rb
 - lib/karafka/schemas/responder_usage.rb
 - lib/karafka/schemas/server_cli_options.rb
 - lib/karafka/server.rb

data/lib/karafka/persistence.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Module used to provide a persistent cache across batch requests for a given
-  # topic and partition to store some additional details when the persistent mode
-  # for a given topic is turned on
-  module Persistence
-    # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
-    # @param partition [Integer] number of partition for which we want to cache
-    # @param resource [Symbol] name of the resource that we want to store
-    def self.fetch(topic, partition, resource)
-      return yield unless topic.persistent
-      Thread.current[topic.id] ||= {}
-      Thread.current[topic.id][partition] ||= {}
-      Thread.current[topic.id][partition][resource] ||= yield
-    end
-  end
-end