karafka 0.6.0.rc2 → 1.0.0.rc1

data/lib/karafka/templates/karafka.rb.example

@@ -17,17 +17,13 @@ class KarafkaApp < Karafka::App
   setup do |config|
     config.kafka.seed_brokers = %w( 127.0.0.1:9092 )
     config.client_id = 'example_app'
-    config.redis = {
-      url: 'redis://localhost:6379'
-    }
-    config.processing_backend = Karafka.env.development? ? :inline : :sidekiq
+    config.backend = :inline
     config.batch_consuming = true
   end
 
   consumer_groups.draw do
     # topic :example do
     #   controller ExampleController
-    #   interchanger CustomInterchanger
     # end
 
     # consumer_group :bigger_group do

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '0.6.0.rc2'
+  VERSION = '1.0.0.rc1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 0.6.0.rc2
+  version: 1.0.0.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-23 00:00:00.000000000 Z
+date: 2017-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
@@ -26,20 +26,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.4'
-- !ruby/object:Gem::Dependency
-  name: sidekiq
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '4.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: celluloid
   requirement: !ruby/object:Gem::Requirement
@@ -210,9 +196,9 @@ files:
 - lib/karafka.rb
 - lib/karafka/app.rb
 - lib/karafka/attributes_map.rb
+- lib/karafka/backends/inline.rb
 - lib/karafka/base_controller.rb
 - lib/karafka/base_responder.rb
-- lib/karafka/base_worker.rb
 - lib/karafka/cli.rb
 - lib/karafka/cli/base.rb
 - lib/karafka/cli/console.rb
@@ -220,11 +206,13 @@ files:
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
-- lib/karafka/cli/worker.rb
 - lib/karafka/connection/config_adapter.rb
 - lib/karafka/connection/listener.rb
 - lib/karafka/connection/messages_consumer.rb
 - lib/karafka/connection/messages_processor.rb
+- lib/karafka/controllers/includer.rb
+- lib/karafka/controllers/responders.rb
+- lib/karafka/controllers/single_params.rb
 - lib/karafka/errors.rb
 - lib/karafka/fetcher.rb
 - lib/karafka/helpers/class_matcher.rb
@@ -233,11 +221,11 @@ files:
 - lib/karafka/loader.rb
 - lib/karafka/logger.rb
 - lib/karafka/monitor.rb
-- lib/karafka/params/interchanger.rb
 - lib/karafka/params/params.rb
 - lib/karafka/params/params_batch.rb
 - lib/karafka/parsers/json.rb
 - lib/karafka/patches/dry_configurable.rb
+- lib/karafka/persistence.rb
 - lib/karafka/process.rb
 - lib/karafka/responders/builder.rb
 - lib/karafka/responders/topic.rb
@@ -255,16 +243,12 @@ files:
 - lib/karafka/setup/config.rb
 - lib/karafka/setup/configurators/base.rb
 - lib/karafka/setup/configurators/celluloid.rb
-- lib/karafka/setup/configurators/sidekiq.rb
 - lib/karafka/setup/configurators/water_drop.rb
 - lib/karafka/status.rb
 - lib/karafka/templates/application_controller.rb.example
 - lib/karafka/templates/application_responder.rb.example
-- lib/karafka/templates/application_worker.rb.example
 - lib/karafka/templates/karafka.rb.example
-- lib/karafka/templates/sidekiq.yml.example
 - lib/karafka/version.rb
-- lib/karafka/workers/builder.rb
 - log/.gitkeep
 homepage: https://github.com/karafka/karafka
 licenses:
@@ -286,7 +270,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka

data/lib/karafka/base_worker.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Worker wrapper for Sidekiq workers
-  class BaseWorker
-    include Sidekiq::Worker
-
-    # Executes the logic that lies in #perform Karafka controller method
-    # @param topic_id [String] Unique topic id that we will use to find a proper topic
-    # @param params_batch [Array] Array with messages batch
-    def perform(topic_id, params_batch)
-      Karafka.monitor.notice(self.class, params_batch)
-      controller(topic_id, params_batch).call
-    end
-
-    private
-
-    # @return [Karafka::Controller] descendant of Karafka::BaseController that matches the topic
-    #   with params_batch assigned already (controller is ready to use)
-    def controller(topic_id, params_batch)
-      @controller ||= Karafka::Routing::Router.build(topic_id).tap do |ctrl|
-        ctrl.params_batch = ctrl.topic.interchanger.parse(params_batch)
-      end
-    end
-  end
-end

data/lib/karafka/cli/worker.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Karafka framework Cli
-  class Cli < Thor
-    # Worker Karafka Cli action
-    class Worker < Base
-      desc 'Start the Karafka Sidekiq worker (short-cut alias: "w")'
-      option aliases: 'w'
-
-      # Start the Karafka Sidekiq worker
-      # @param params [Array<String>] additional params that will be passed to sidekiq, that way we
-      #   can override any default Karafka settings
-      def call(*params)
-        puts 'Starting Karafka worker'
-        config = "-C #{Karafka::App.root.join('config/sidekiq.yml')}"
-        req = "-r #{Karafka.boot_file}"
-        env = "-e #{Karafka.env}"
-
-        cli.info
-
-        cmd = "bundle exec sidekiq #{env} #{req} #{config} #{params.join(' ')}"
-        puts(cmd)
-        exec(cmd)
-      end
-    end
-  end
-end

data/lib/karafka/params/interchanger.rb

@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    # Interchangers allow us to format/encode/pack data that is being send to perform_async
-    # This is meant to target mostly issues with data encoding like this one:
-    # https://github.com/mperham/sidekiq/issues/197
-    # Each custom interchanger should implement following methods:
-    #   - load - it is meant to encode params before they get stored inside Redis
-    #   - parse - decoded params back to a hash format that we can use
-    class Interchanger
-      class << self
-        # @param params [Karafka::Params::Params] Karafka params object
-        # @note Params might not be parsed because of lazy loading feature. If you implement your
-        #   own interchanger logic, this method needs to return data that can be converted to
-        #   json with default Sidekiqs logic
-        # @return [Karafka::Params::Params] same as input. We assume that our incoming data is
-        #   jsonable-safe and we can rely on a direct Sidekiq encoding logic
-        def load(params)
-          params
-        end
-
-        # @param params [Hash] Sidekiqs params that are now a Hash (after they were JSON#parse)
-        # @note Hash is what we need to build Karafka::Params::Params, so we do nothing
-        #   with it. If you implement your own interchanger logic, this method needs to return
-        #   a hash with appropriate data that will be used to build Karafka::Params::Params
-        # @return [Hash] We return exactly what we received. We rely on sidekiqs default
-        #   interchanging format
-        def parse(params)
-          params
-        end
-      end
-    end
-  end
-end

data/lib/karafka/setup/configurators/sidekiq.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Setup
-    class Configurators
-      # Class to configure all the Sidekiq settings based on Karafka settings
-      class Sidekiq < Base
-        # Sets up sidekiq client and server
-        def setup
-          setup_sidekiq_client
-          setup_sidekiq_server
-        end
-
-        private
-
-        # Configure sidekiq client
-        def setup_sidekiq_client
-          ::Sidekiq.configure_client do |sidekiq_config|
-            sidekiq_config.redis = config.redis.to_h.merge(
-              size: config.concurrency
-            )
-          end
-        end
-
-        # Configure sidekiq setorrver
-        def setup_sidekiq_server
-          ::Sidekiq.configure_server do |sidekiq_config|
-            # We don't set size for the server - this will be set automatically based
-            # on the Sidekiq concurrency level (Sidekiq not Karafkas)
-            sidekiq_config.redis = config.redis.to_h
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/templates/application_worker.rb.example

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-# Application worker from which all workers should inherit
-# You can rename it if it would conflict with your current code base (in case you're integrating
-# Karafka with other frameworks). Karafka will use first direct descendant of Karafka::BaseWorker
-# to build worker classes
-class ApplicationWorker < Karafka::BaseWorker
-end

data/lib/karafka/templates/sidekiq.yml.example

@@ -1,26 +0,0 @@
-# Sample configuration file for Sidekiq.
-# Options here can still be overridden by cmd line args.
-# Place this file at config/sidekiq.yml and Sidekiq will
-# pick it up automatically.
-
-:verbose: true
-:concurrency: 10
-
-# Set timeout to 8 on Heroku, longer if you manage your own systems.
-:timeout: 8
-
-# Sidekiq will run this file through ERB when reading it so you can
-# even put in dynamic logic, like a host-specific queue.
-## http://www.mikeperham.com/2013/11/13/advanced-sidekiq-host-specific-queues/
-#:queues:
-#  - all
-#  - high
-#  - highest321
-
-# you can override concurrency based on environment
-production:
-  :concurrency: 10
-development:
-  :concurrency: 1
-test:
-  :concurrency: 1

data/lib/karafka/workers/builder.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Internal stuff related to workers
-  module Workers
-    # Builder is used to check if there is a proper controller with the same name as
-    # a controller and if not, it will create a default one using Karafka::BaseWorker
-    # This is used as a building layer between controllers and workers. it will be only used
-    # when user does not provide his own worker that should perform controller stuff
-    class Builder
-      # @param controller_class [Karafka::BaseController] descendant of Karafka::BaseController
-      # @example Create a worker builder
-      #   Karafka::Workers::Builder.new(SuperController)
-      def initialize(controller_class)
-        @controller_class = controller_class
-      end
-
-      # @return [Class] Sidekiq worker class that already exists or new build based
-      #   on the provided controller_class name
-      # @example Controller: SuperController
-      #   build #=> SuperWorker
-      # @example Controller: Videos::NewVideosController
-      #   build #=> Videos::NewVideosWorker
-      def build
-        return matcher.match if matcher.match
-        klass = Class.new(base)
-        matcher.scope.const_set(matcher.name, klass)
-      end
-
-      private
-
-      # @return [Class] descendant of Karafka::BaseWorker from which all other workers
-      #   should inherit
-      # @raise [Karafka::Errors::BaseWorkerDescentantMissing] raised when Karafka cannot detect
-      #   direct Karafka::BaseWorker descendant from which it could build workers
-      def base
-        Karafka::BaseWorker.subclasses.first || raise(Errors::BaseWorkerDescentantMissing)
-      end
-
-      # @return [Karafka::Helpers::ClassMatcher] matcher instance for matching between controller
-      #   and appropriate worker
-      def matcher
-        @matcher ||= Helpers::ClassMatcher.new(
-          @controller_class,
-          from: 'Controller',
-          to: 'Worker'
-        )
-      end
-    end
-  end
-end