karafka 1.0.1 → 1.1.0.alpha1

data/lib/karafka/responders/topic.rb

@@ -36,13 +36,20 @@ module Karafka
         @options[:registered] == true
       end
 
+      # @return [Boolean] do we want to use async producer. Defaults to false as the sync producer
+      #   is safer and introduces less problems
+      def async?
+        @options.key?(:async) ? @options[:async] : false
+      end
+
       # @return [Hash] hash with this topic attributes and options
       def to_h
         {
           name: name,
           multiple_usage: multiple_usage?,
           required: required?,
-          registered: registered?
+          registered: registered?,
+          async: async?
         }
       end
     end

data/lib/karafka/schemas/config.rb

@@ -15,17 +15,7 @@ module Karafka
       required(:client_id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
       required(:consumer_mapper)
       required(:topic_mapper)
-
-      required(:celluloid).schema do
-        required(:shutdown_timeout).filled(:int?, gteq?: 0)
-      end
-
       optional(:backend).filled
-
-      optional(:connection_pool).schema do
-        required(:size).filled
-        optional(:timeout).filled(:int?)
-      end
     end
   end
 end

data/lib/karafka/schemas/consumer_group.rb

@@ -27,16 +27,17 @@ module Karafka
 
       required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
       required(:seed_brokers).filled { each(:broker_schema?) }
-      required(:session_timeout).filled(:int?)
-      required(:pause_timeout).filled(:int?, gteq?: 0)
-      required(:offset_commit_interval).filled(:int?)
+      required(:session_timeout).filled { int? | float? }
+      required(:pause_timeout).filled { (int? | float?) & gteq?(0) }
+      required(:offset_commit_interval) { int? | float? }
       required(:offset_commit_threshold).filled(:int?)
       required(:offset_retention_time) { none?.not > int? }
-      required(:heartbeat_interval).filled(:int?, gteq?: 0)
-      required(:connect_timeout).filled(:int?, gt?: 0)
-      required(:socket_timeout).filled(:int?, gt?: 0)
-      required(:max_wait_time).filled(:int?, gteq?: 0)
-      required(:batch_consuming).filled(:bool?)
+      required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
+      required(:connect_timeout).filled { (int? | float?) & gt?(0) }
+      required(:socket_timeout).filled { (int? | float?) & gt?(0) }
+      required(:min_bytes).filled(:int?, gt?: 0)
+      required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
+      required(:batch_fetching).filled(:bool?)
       required(:topics).filled { each { schema(ConsumerGroupTopic) } }
 
       # Max wait time cannot exceed socket_timeout - wouldn't make sense

data/lib/karafka/schemas/consumer_group_topic.rb

@@ -11,7 +11,7 @@ module Karafka
       required(:parser).filled
       required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
       required(:start_from_beginning).filled(:bool?)
-      required(:batch_processing).filled(:bool?)
+      required(:batch_consuming).filled(:bool?)
       required(:persistent).filled(:bool?)
     end
   end

data/lib/karafka/schemas/responder_usage.rb

@@ -9,6 +9,7 @@ module Karafka
       required(:multiple_usage).filled(:bool?)
       required(:usage_count).filled(:int?, gteq?: 0)
       required(:registered).filled(eql?: true)
+      required(:async).filled(:bool?)
 
       rule(
         required_usage: %i[required usage_count]

data/lib/karafka/server.rb

@@ -4,16 +4,15 @@ module Karafka
   # Karafka consuming server class
   class Server
     class << self
-      # We need to store reference to all the consumers in the main server thread,
-      # So we can have access to them later on and be able to stop them on exit
-      attr_reader :consumers
+      # Set of consuming threads. Each consumer thread contains a single consumer
+      attr_accessor :consumer_threads
 
       # Writer for list of consumer groups that we want to consume in our current process context
       attr_writer :consumer_groups
 
       # Method which runs app
       def run
-        @consumers = Concurrent::Array.new
+        @consumer_threads = Concurrent::Array.new
         bind_on_sigint
         bind_on_sigquit
         bind_on_sigterm
@@ -36,29 +35,17 @@ module Karafka
 
       # What should happen when we decide to quit with sigint
       def bind_on_sigint
-        process.on_sigint do
-          Karafka::App.stop!
-          consumers.map(&:stop)
-          Kernel.exit
-        end
+        process.on_sigint { Karafka::App.stop! }
       end
 
       # What should happen when we decide to quit with sigquit
       def bind_on_sigquit
-        process.on_sigquit do
-          Karafka::App.stop!
-          consumers.map(&:stop)
-          Kernel.exit
-        end
+        process.on_sigquit { Karafka::App.stop! }
       end
 
       # What should happen when we decide to quit with sigterm
       def bind_on_sigterm
-        process.on_sigterm do
-          Karafka::App.stop!
-          consumers.map(&:stop)
-          Kernel.exit
-        end
+        process.on_sigterm { Karafka::App.stop! }
       end
 
       # Starts Karafka with a supervision

data/lib/karafka/setup/config.rb

@@ -34,42 +34,19 @@ module Karafka
       #   - #incoming - for remapping from the incoming message to our internal format
       #   - #outgoing - for remapping from internal topic name into outgoing message
       setting :topic_mapper, -> { Routing::TopicMapper }
-      # If batch_consuming is true, we will consume kafka messages in batches instead of 1 by 1
-      # @note Consuming does not equal processing, see batch_processing description for details
-      setting :batch_consuming, true
-      # If batch_processing is true, we will have access to #params_batch instead of #params.
+      # If batch_fetching is true, we will fetch kafka messages in batches instead of 1 by 1
+      # @note Fetching does not equal consuming, see batch_consuming description for details
+      setting :batch_fetching, true
+      # If batch_consuming is true, we will have access to #params_batch instead of #params.
       # #params_batch will contain params received from Kafka (may be more than 1) so we can
       # process them in batches
-      setting :batch_processing, false
+      setting :batch_consuming, false
       # Should we operate in a single controller instance across multiple batches of messages,
       # from the same partition or should we build a new instance for each incoming batch.
       # Disabling that can be useful when you want to build a new controller instance for each
       # incoming batch. It's disabled by default, not to create more objects that needed on
       # each batch
       setting :persistent, true
-      # This is configured automatically, don't overwrite it!
-      # Each consumer group requires separate thread, so number of threads should be equal to
-      # number of consumer groups
-      setting :concurrency, -> { ::Karafka::App.consumer_groups.count }
-
-      # option celluloid [Hash] - optional - celluloid configuration options
-      setting :celluloid do
-        # options shutdown_timeout [Integer] How many seconds should we wait for actors (listeners)
-        # before forcefully shutting them
-        setting :shutdown_timeout, 30
-      end
-
-      # Connection pool options are used for producer (Waterdrop) - by default it will adapt to
-      # number of active actors
-      setting :connection_pool do
-        # Connection pool size for producers. If you use sidekiq or any other multi threaded
-        # backend, you might want to tune it to match number of threads of your background
-        # processing engine
-        setting :size, -> { ::Karafka::App.consumer_groups.active.count }
-        # How long should we wait for a working resource from the pool before rising timeout
-        # With a proper connection pool size, this should never happen
-        setting :timeout, 5
-      end
 
       # option kafka [Hash] - optional - kafka configuration options
       setting :kafka do
@@ -78,17 +55,17 @@ module Karafka
         # option session_timeout [Integer] the number of seconds after which, if a client
         #   hasn't contacted the Kafka cluster, it will be kicked out of the group.
         setting :session_timeout, 30
-        # Time that a given partition will be paused from processing messages, when message
-        # processing fails. It allows us to process other partitions, while the error is being
+        # Time that a given partition will be paused from fetching messages, when message
+        # consumption fails. It allows us to process other partitions, while the error is being
         # resolved and also "slows" things down, so it prevents from "eating" up all messages and
-        # processing them with failed code
+        # consuming them with failed code
         setting :pause_timeout, 10
         # option offset_commit_interval [Integer] the interval between offset commits,
         #   in seconds.
         setting :offset_commit_interval, 10
         # option offset_commit_threshold [Integer] the number of messages that can be
         #   processed before their offsets are committed. If zero, offset commits are
-        #   not triggered by message processing.
+        #   not triggered by message consumption.
         setting :offset_commit_threshold, 0
         # option heartbeat_interval [Integer] the interval between heartbeats; must be less
         #   than the session window.
@@ -104,12 +81,15 @@ module Karafka
         setting :min_bytes, 1
         # option max_wait_time [Integer, Float] max_wait_time is the maximum number of seconds to
         #   wait before returning data from a single message fetch. By setting this high you also
-        #   increase the processing throughput – and by setting it low you set a bound on latency.
+        #   increase the fetching throughput - and by setting it low you set a bound on latency.
         #   This configuration overrides `min_bytes`, so you'll _always_ get data back within the
         #   time specified. The default value is one second. If you want to have at most five
         #   seconds of latency, set `max_wait_time` to 5. You should make sure
         #   max_wait_time * num brokers + heartbeat_interval is less than session_timeout.
         setting :max_wait_time, 1
+        # option automatically_mark_as_processed [Boolean] should we automatically mark received
+        # messages as processed after non-error consumption
+        setting :automatically_mark_as_processed, true
         # option reconnect_timeout [Integer] How long should we wait before trying to reconnect to
         # Kafka cluster that went down (in seconds)
         setting :reconnect_timeout, 5
@@ -124,7 +104,8 @@ module Karafka
         # writing to a socket connection to a broker. After this timeout expires the connection
         # will be killed. Note that some Kafka operations are by definition long-running, such as
         # waiting for new messages to arrive in a partition, so don't set this value too low
-        setting :socket_timeout, 10
+        setting :socket_timeout, 30
+
         # SSL authentication related settings
         # option ca_cert [String] SSL CA certificate
         setting :ssl_ca_cert, nil

data/lib/karafka/setup/configurators/base.rb

@@ -4,7 +4,7 @@ module Karafka
   module Setup
     # Configurators module is used to enclose all the external dependencies configurations
     class Configurators
-      # Karafka has come components that it relies on (like Celluloid or Sidekiq)
+      # Karafka has come components that it relies on (like Sidekiq)
       # We need to configure all of them only when the framework was set up.
       # Any class that descends from this one will be automatically invoked upon setup (after it)
       # @example Configure an Example class

data/lib/karafka/setup/configurators/water_drop.rb

@@ -7,21 +7,19 @@ module Karafka
       class WaterDrop < Base
         # Sets up a WaterDrop settings
         def setup
-          dynamic_params = Connection::ConfigAdapter.client(nil)
-
           ::WaterDrop.setup do |water_config|
-            water_config.send_messages = true
-            water_config.raise_on_failure = true
-            water_config.connection_pool = config.connection_pool
+            water_config.deliver = true
+
+            Karafka::App.config.to_h.except(:kafka).each do |k, v|
+              key_assignment = :"#{k}="
+              next unless water_config.respond_to?(key_assignment)
+              water_config.public_send(key_assignment, v)
+            end
 
-            # Automigration of all the attributes that should be accepted by waterdrop
-            # based on what we use in karafka ruby-kafka initialization
-            dynamic_params.each do |key, value|
-              key_assignment = :"#{key}="
-              # We decide whether we should set it on a kafka scope of waterdrop config or on the
-              # main scope
-              scope = water_config.kafka.respond_to?(key_assignment) ? :kafka : :itself
-              water_config.public_send(scope).public_send(key_assignment, value)
+            Karafka::App.config.kafka.to_h.each do |k, v|
+              key_assignment = :"#{k}="
+              next unless water_config.kafka.respond_to?(key_assignment)
+              water_config.kafka.public_send(key_assignment, v)
             end
           end
         end

data/lib/karafka/templates/karafka.rb.example

@@ -18,7 +18,7 @@ class KarafkaApp < Karafka::App
     config.kafka.seed_brokers = %w( 127.0.0.1:9092 )
     config.client_id = 'example_app'
     config.backend = :inline
-    config.batch_consuming = true
+    config.batch_fetching = true
   end
 
   consumer_groups.draw do

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.0.1'
+  VERSION = '1.1.0.alpha1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0.alpha1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-27 00:00:00.000000000 Z
+date: 2017-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -26,20 +26,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: celluloid
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: dry-configurable
   requirement: !ruby/object:Gem::Requirement
@@ -130,14 +116,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -156,16 +142,16 @@ dependencies:
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 1.0.alpha2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 1.0.alpha2
 description: Framework used to simplify Apache Kafka based Ruby applications development
 email:
 - maciej@coditsu.io
@@ -190,7 +176,6 @@ files:
 - Gemfile.lock
 - MIT-LICENCE
 - README.md
-- Rakefile
 - bin/karafka
 - config/errors.yml
 - karafka.gemspec
@@ -208,9 +193,10 @@ files:
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
 - lib/karafka/connection/config_adapter.rb
+- lib/karafka/connection/consumer.rb
 - lib/karafka/connection/listener.rb
-- lib/karafka/connection/messages_consumer.rb
-- lib/karafka/connection/messages_processor.rb
+- lib/karafka/connection/processor.rb
+- lib/karafka/controllers/callbacks.rb
 - lib/karafka/controllers/includer.rb
 - lib/karafka/controllers/responders.rb
 - lib/karafka/controllers/single_params.rb
@@ -226,6 +212,8 @@ files:
 - lib/karafka/params/params_batch.rb
 - lib/karafka/parsers/json.rb
 - lib/karafka/patches/dry_configurable.rb
+- lib/karafka/patches/ruby_kafka.rb
+- lib/karafka/persistence/consumer.rb
 - lib/karafka/persistence/controller.rb
 - lib/karafka/process.rb
 - lib/karafka/responders/builder.rb
@@ -245,7 +233,6 @@ files:
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb
 - lib/karafka/setup/configurators/base.rb
-- lib/karafka/setup/configurators/celluloid.rb
 - lib/karafka/setup/configurators/water_drop.rb
 - lib/karafka/status.rb
 - lib/karafka/templates/application_controller.rb.example
@@ -268,9 +255,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.13

data/Rakefile

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-require 'rspec'
-require 'rspec/core/rake_task'
-
-RSpec::Core::RakeTask.new(:spec)
-task default: :spec

data/lib/karafka/setup/configurators/celluloid.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Setup
-    class Configurators
-      # Class responsible for setting up Celluloid settings
-      class Celluloid < Base
-        # Sets up a Karafka logger as celluloid logger
-        def setup
-          ::Celluloid.logger = ::Karafka.logger
-          # This is just a precaution - it should automatically close the current
-          # connection and shutdown actor - but in case it didn't (hanged, etc)
-          # we will kill it after waiting for some time
-          ::Celluloid.shutdown_timeout = config.celluloid.shutdown_timeout
-        end
-      end
-    end
-  end
-end